Options

[SOLVED] Error:Deviaion of performance was NAN

knnbayazknnbayaz Member Posts: 9 Contributor II
edited November 2018 in Help
Hi

I used evolutionary optimization operator to optimize select by weights operator and nu-SVR at the same time. but in the log wiev, i encountered the error "PM WARNING: SimpleCriterion: Deviation of Performance was NaN!". How can i fix the problem? Please i am waiting for your help

My model xml are: 
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.2.008">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="5.2.008" expanded="true" name="Process">
    <parameter key="parallelize_main_process" value="true"/>
    <process expanded="true" height="400" width="480">
      <operator activated="true" class="retrieve" compatibility="5.2.008" expanded="true" height="60" name="Retrieve" width="90" x="45" y="30">
        <parameter key="repository_entry" value="//kenan/Tarih/20.11.2012/Clus3/ClusByKMeans(Fast)Euclidian"/>
      </operator>
      <operator activated="true" class="weight_by_relief" compatibility="5.2.008" expanded="true" height="76" name="Weight by Relief" width="90" x="180" y="30"/>
      <operator activated="true" class="parallel:optimize_parameters_evolutionary_parallel" compatibility="5.1.000" expanded="true" height="148" name="Optimize Parameters (Evolutionary)" width="90" x="313" y="210">
        <list key="parameters">
          <parameter key="Select by Weights (2).k" value="[15;35]"/>
          <parameter key="SVM (2).gamma" value="[0.0;0.01]"/>
          <parameter key="SVM (2).nu" value="[0.0;0.5]"/>
          <parameter key="SVM (2).C" value="[0.0;0.01]"/>
          <parameter key="SVM (2).epsilon" value="[0.0;0.01]"/>
        </list>
        <parameter key="max_generations" value="90"/>
        <parameter key="population_size" value="6"/>
        <parameter key="crossover_prob" value="0.8"/>
        <parameter key="number_of_threads" value="8"/>
        <parameter key="parallelize_optimization_process" value="true"/>
        <process expanded="true" height="400" width="632">
          <operator activated="true" class="select_by_weights" compatibility="5.2.008" expanded="true" height="94" name="Select by Weights (2)" width="90" x="45" y="30">
            <parameter key="weight_relation" value="top k"/>
            <parameter key="k" value="16"/>
          </operator>
          <operator activated="true" class="split_validation" compatibility="5.1.002" expanded="true" height="112" name="Validation (2)" width="90" x="380" y="30">
            <parameter key="split_ratio" value="0.9"/>
            <parameter key="sampling_type" value="linear sampling"/>
            <parameter key="parallelize_training" value="true"/>
            <parameter key="parallelize_testing" value="true"/>
            <process expanded="true" height="400" width="165">
              <operator activated="true" class="support_vector_machine_libsvm" compatibility="5.2.008" expanded="true" height="76" name="SVM (2)" width="90" x="45" y="30">
                <parameter key="svm_type" value="nu-SVR"/>
                <parameter key="gamma" value="0.0018245416895055634"/>
                <parameter key="C" value="0.00562070136566738"/>
                <parameter key="nu" value="0.48424641085734477"/>
                <parameter key="epsilon" value="0.009366372796859642"/>
                <list key="class_weights"/>
              </operator>
              <connect from_port="training" to_op="SVM (2)" to_port="training set"/>
              <connect from_op="SVM (2)" from_port="model" to_port="model"/>
              <portSpacing port="source_training" spacing="0"/>
              <portSpacing port="sink_model" spacing="0"/>
              <portSpacing port="sink_through 1" spacing="0"/>
            </process>
            <process expanded="true" height="400" width="300">
              <operator activated="true" class="apply_model" compatibility="5.2.008" expanded="true" height="76" name="Apply Model (2)" width="90" x="45" y="30">
                <list key="application_parameters"/>
              </operator>
              <operator activated="true" class="performance_regression" compatibility="5.2.008" expanded="true" height="76" name="Performance (2)" width="90" x="180" y="30">
                <parameter key="main_criterion" value="root_mean_squared_error"/>
                <parameter key="absolute_error" value="true"/>
                <parameter key="relative_error" value="true"/>
                <parameter key="root_relative_squared_error" value="true"/>
                <parameter key="squared_error" value="true"/>
                <parameter key="squared_correlation" value="true"/>
                <parameter key="spearman_rho" value="true"/>
                <parameter key="kendall_tau" value="true"/>
              </operator>
              <connect from_port="model" to_op="Apply Model (2)" to_port="model"/>
              <connect from_port="test set" to_op="Apply Model (2)" to_port="unlabelled data"/>
              <connect from_op="Apply Model (2)" from_port="labelled data" to_op="Performance (2)" to_port="labelled data"/>
              <connect from_op="Performance (2)" from_port="performance" to_port="averagable 1"/>
              <portSpacing port="source_model" spacing="0"/>
              <portSpacing port="source_test set" spacing="0"/>
              <portSpacing port="source_through 1" spacing="0"/>
              <portSpacing port="sink_averagable 1" spacing="0"/>
              <portSpacing port="sink_averagable 2" spacing="0"/>
            </process>
          </operator>
          <connect from_port="input 1" to_op="Select by Weights (2)" to_port="weights"/>
          <connect from_port="input 2" to_op="Select by Weights (2)" to_port="example set input"/>
          <connect from_op="Select by Weights (2)" from_port="example set output" to_op="Validation (2)" to_port="training"/>
          <connect from_op="Select by Weights (2)" from_port="weights" to_port="result 3"/>
          <connect from_op="Validation (2)" from_port="model" to_port="result 1"/>
          <connect from_op="Validation (2)" from_port="training" to_port="result 2"/>
          <connect from_op="Validation (2)" from_port="averagable 1" to_port="performance"/>
          <portSpacing port="source_input 1" spacing="0"/>
          <portSpacing port="source_input 2" spacing="0"/>
          <portSpacing port="source_input 3" spacing="0"/>
          <portSpacing port="sink_performance" spacing="0"/>
          <portSpacing port="sink_result 1" spacing="0"/>
          <portSpacing port="sink_result 2" spacing="0"/>
          <portSpacing port="sink_result 3" spacing="0"/>
          <portSpacing port="sink_result 4" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="log" compatibility="5.2.008" expanded="true" height="76" name="Log" width="90" x="380" y="30">
        <parameter key="filename" value="C:\Users\KenanB\Desktop\RapidminerLog\ParameterLog.txt"/>
        <list key="log">
          <parameter key="ApplyCount" value="operator.Optimize Parameters (Evolutionary).value.applycount"/>
          <parameter key="RMSE" value="operator.Performance (2).value.root_mean_squared_error"/>
          <parameter key="K" value="operator.Select by Weights (2).parameter.k"/>
          <parameter key="C" value="operator.SVM (2).parameter.C"/>
          <parameter key="NU" value="operator.SVM (2).parameter.nu"/>
          <parameter key="Gamma" value="operator.SVM (2).parameter.gamma"/>
          <parameter key="Epsilon" value="operator.SVM (2).parameter.epsilon"/>
        </list>
      </operator>
      <connect from_op="Retrieve" from_port="output" to_op="Weight by Relief" to_port="example set"/>
      <connect from_op="Weight by Relief" from_port="weights" to_op="Optimize Parameters (Evolutionary)" to_port="input 1"/>
      <connect from_op="Weight by Relief" from_port="example set" to_op="Optimize Parameters (Evolutionary)" to_port="input 2"/>
      <connect from_op="Optimize Parameters (Evolutionary)" from_port="performance" to_port="result 2"/>
      <connect from_op="Optimize Parameters (Evolutionary)" from_port="parameter" to_op="Log" to_port="through 1"/>
      <connect from_op="Optimize Parameters (Evolutionary)" from_port="result 1" to_port="result 3"/>
      <connect from_op="Optimize Parameters (Evolutionary)" from_port="result 2" to_port="result 4"/>
      <connect from_op="Log" from_port="through 1" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
      <portSpacing port="sink_result 3" spacing="0"/>
      <portSpacing port="sink_result 4" spacing="0"/>
      <portSpacing port="sink_result 5" spacing="0"/>
    </process>
  </operator>
</process>

Answers

  • Options
    MariusHelfMariusHelf RapidMiner Certified Expert, Member Posts: 1,869 Unicorn
    Hi,

    as written elsewhere the evolutionary optimization operators have some hiccups at times. Please consider using the grid parameter optimization instead. Your process setup looks fine!

    Good luck,
    Marius
  • Options
    knnbayazknnbayaz Member Posts: 9 Contributor II
    Hi Marcus

    Thank for your reply. As you said there are really hiccups in evolutionary optimization. I clustered the data into three clusters. For two clusters this process works very vell. I watched log when processing, there was only two or three times i saw deviation was nan error. The process worked well and i got the results. For last cluster i got this error message. when i was watching log, sometimes i saw a message like "compatibility level of split validation was changed to 5.0.000". I think the cause of error was compatibility problem. Do you have any idea about compatibility problem? I need evolutionary optimization because the range for optimizing is too high. Grid optimization will be very slow.

    Thanks again for your concern
  • Options
    MariusHelfMariusHelf RapidMiner Certified Expert, Member Posts: 1,869 Unicorn
    Hi,

    I don't know if it will solve your problems, but I have some remarks on your process setup:

    - The Split Validation is very dependent on the split point and much less robust than the X-Validation. You selected linear sampling, which means that your training set will always be the first 90 percent of the examples. If your data is sorted in any way, that will be a problem.
    - Your Split Validation is set to compatibility level 5.1.002. Try to increase it to the current version of RapidMiner.
    - Please inspect the problematic cluster manually. Do both training and test set contain reasonable examples?

    Concerning the log, please post the errors with some lines before and behind. You should also try to start RapidMiner from the command line and see if you see further messages in the console.

    ~Marius
  • Options
    knnbayazknnbayaz Member Posts: 9 Contributor II
    Hi

    Thank you for your recommendations. I have renewed the model. I added the log operator after the validation. now it is working. Before running, ı worked it in debug mode. This way solved my problem but i don't think it is a robust solution. I am looking forward to develop the optmization operator from rapidminer team. I think it is best part of rapidminer. Thanks again for your instructions. My new model xml is below: 
    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
    <process version="5.2.008">
      <context>
        <input/>
        <output/>
        <macros/>
      </context>
      <operator activated="true" class="process" compatibility="5.2.008" expanded="true" name="Process">
        <parameter key="parallelize_main_process" value="true"/>
        <process expanded="true" height="271" width="570">
          <operator activated="true" class="retrieve" compatibility="5.2.008" expanded="true" height="60" name="Retrieve" width="90" x="45" y="30">
            <parameter key="repository_entry" value="ClusByKMeans(Fast)Euclidian"/>
          </operator>
          <operator activated="true" class="subprocess" compatibility="5.2.008" expanded="true" height="112" name="Subprocess" width="90" x="180" y="30">
            <process expanded="true" height="387" width="705">
              <operator activated="true" class="set_role" compatibility="5.2.008" expanded="true" height="76" name="Set Role" width="90" x="45" y="30">
                <parameter key="name" value="cluster"/>
                <list key="set_additional_roles"/>
              </operator>
              <operator activated="true" class="multiply" compatibility="5.2.008" expanded="true" height="112" name="Multiply" width="90" x="45" y="165"/>
              <operator activated="true" class="filter_examples" compatibility="5.2.008" expanded="true" height="76" name="Filter Examples (3)" width="90" x="246" y="210">
                <parameter key="condition_class" value="attribute_value_filter"/>
                <parameter key="parameter_string" value="cluster=cluster_2"/>
              </operator>
              <operator activated="true" class="select_attributes" compatibility="5.2.008" expanded="true" height="76" name="Select Attributes (3)" width="90" x="380" y="210">
                <parameter key="attribute_filter_type" value="subset"/>
                <parameter key="attributes" value="|Çarşamba|WeekLag1|Time|Salı|SaatOrtalama1|SaatEksiSapma1|SaatArtıSapma1|SGOF|Perşembe|Pazartesi|Pazar|GunLag2|GunLag1|GunHoliday|Cumartesi|Cuma|23:00|22:00|21:00|20:00|19:00|18:00|17:00|16:00|15:00|14:00|13:00|12:00|11:00|10:00|09:00|08:00|07:00|06:00|05:00|04:00|03:00|02:00|01:00|00:00"/>
              </operator>
              <operator activated="true" class="filter_examples" compatibility="5.2.008" expanded="true" height="76" name="Filter Examples (2)" width="90" x="246" y="120">
                <parameter key="condition_class" value="attribute_value_filter"/>
                <parameter key="parameter_string" value="cluster=cluster_1"/>
              </operator>
              <operator activated="true" class="select_attributes" compatibility="5.2.008" expanded="true" height="76" name="Select Attributes (2)" width="90" x="380" y="120">
                <parameter key="attribute_filter_type" value="subset"/>
                <parameter key="attributes" value="|Çarşamba|WeekLag1|Time|Salı|SaatOrtalama1|SaatEksiSapma1|SaatArtıSapma1|SGOF|Perşembe|Pazartesi|Pazar|GunLag2|GunLag1|GunHoliday|Cumartesi|Cuma|23:00|22:00|21:00|20:00|19:00|18:00|17:00|16:00|15:00|14:00|13:00|12:00|11:00|10:00|09:00|08:00|07:00|06:00|05:00|04:00|03:00|02:00|01:00|00:00"/>
              </operator>
              <operator activated="true" class="filter_examples" compatibility="5.2.008" expanded="true" height="76" name="Filter Examples" width="90" x="246" y="30">
                <parameter key="condition_class" value="attribute_value_filter"/>
                <parameter key="parameter_string" value="cluster=cluster_0"/>
              </operator>
              <operator activated="true" class="select_attributes" compatibility="5.2.008" expanded="true" height="76" name="Select Attributes" width="90" x="380" y="30">
                <parameter key="attribute_filter_type" value="subset"/>
                <parameter key="attributes" value="|Çarşamba|WeekLag1|Time|Salı|SaatOrtalama1|SaatEksiSapma1|SaatArtıSapma1|SGOF|Perşembe|Pazartesi|Pazar|GunLag2|GunLag1|GunHoliday|Cumartesi|Cuma|23:00|22:00|21:00|20:00|19:00|18:00|17:00|16:00|15:00|14:00|13:00|12:00|11:00|10:00|09:00|08:00|07:00|06:00|05:00|04:00|03:00|02:00|01:00|00:00"/>
              </operator>
              <connect from_port="in 1" to_op="Set Role" to_port="example set input"/>
              <connect from_op="Set Role" from_port="example set output" to_op="Multiply" to_port="input"/>
              <connect from_op="Multiply" from_port="output 1" to_op="Filter Examples" to_port="example set input"/>
              <connect from_op="Multiply" from_port="output 2" to_op="Filter Examples (2)" to_port="example set input"/>
              <connect from_op="Multiply" from_port="output 3" to_op="Filter Examples (3)" to_port="example set input"/>
              <connect from_op="Filter Examples (3)" from_port="example set output" to_op="Select Attributes (3)" to_port="example set input"/>
              <connect from_op="Select Attributes (3)" from_port="example set output" to_port="out 3"/>
              <connect from_op="Filter Examples (2)" from_port="example set output" to_op="Select Attributes (2)" to_port="example set input"/>
              <connect from_op="Select Attributes (2)" from_port="example set output" to_port="out 2"/>
              <connect from_op="Filter Examples" from_port="example set output" to_op="Select Attributes" to_port="example set input"/>
              <connect from_op="Select Attributes" from_port="example set output" to_port="out 1"/>
              <portSpacing port="source_in 1" spacing="0"/>
              <portSpacing port="source_in 2" spacing="0"/>
              <portSpacing port="sink_out 1" spacing="0"/>
              <portSpacing port="sink_out 2" spacing="0"/>
              <portSpacing port="sink_out 3" spacing="0"/>
              <portSpacing port="sink_out 4" spacing="0"/>
            </process>
          </operator>
          <operator activated="true" class="weight_by_relief" compatibility="5.2.008" expanded="true" height="76" name="Weight by Relief" width="90" x="315" y="30"/>
          <operator activated="true" class="parallel:optimize_parameters_evolutionary_parallel" compatibility="5.1.000" expanded="true" height="130" name="Optimize Parameters (Evolutionary)" width="90" x="450" y="30">
            <list key="parameters">
              <parameter key="Select by Weights.k" value="[10;35]"/>
              <parameter key="SVM.C" value="[1E-6;1E-3]"/>
              <parameter key="SVM.nu" value="[0.0;0.5]"/>
              <parameter key="SVM.epsilon" value="[0.0001;0.01]"/>
              <parameter key="SVM.gamma" value="[0.0001;0.0009]"/>
            </list>
            <parameter key="max_generations" value="100"/>
            <parameter key="population_size" value="6"/>
            <parameter key="crossover_prob" value="0.8"/>
            <parameter key="number_of_threads" value="8"/>
            <parameter key="parallelize_optimization_process" value="true"/>
            <process expanded="true" height="271" width="547">
              <operator activated="true" class="select_by_weights" compatibility="5.2.008" expanded="true" height="94" name="Select by Weights" width="90" x="45" y="30">
                <parameter key="weight_relation" value="top k"/>
                <parameter key="k" value="22"/>
              </operator>
              <operator activated="true" class="split_validation" compatibility="5.2.008" expanded="true" height="112" name="Validation" width="90" x="246" y="30">
                <parameter key="split_ratio" value="0.9"/>
                <parameter key="parallelize_training" value="true"/>
                <parameter key="parallelize_testing" value="true"/>
                <process expanded="true" height="289" width="307">
                  <operator activated="true" class="support_vector_machine_libsvm" compatibility="5.2.008" expanded="true" height="76" name="SVM" width="90" x="112" y="30">
                    <parameter key="svm_type" value="nu-SVR"/>
                    <parameter key="gamma" value="1.0E-4"/>
                    <parameter key="C" value="0.0010"/>
                    <parameter key="nu" value="0.49301039536565583"/>
                    <parameter key="epsilon" value="0.006058143225658175"/>
                    <list key="class_weights"/>
                  </operator>
                  <connect from_port="training" to_op="SVM" to_port="training set"/>
                  <connect from_op="SVM" from_port="model" to_port="model"/>
                  <portSpacing port="source_training" spacing="0"/>
                  <portSpacing port="sink_model" spacing="0"/>
                  <portSpacing port="sink_through 1" spacing="0"/>
                </process>
                <process expanded="true" height="289" width="307">
                  <operator activated="true" class="apply_model" compatibility="5.2.008" expanded="true" height="76" name="Apply Model" width="90" x="45" y="30">
                    <list key="application_parameters"/>
                  </operator>
                  <operator activated="true" class="performance_regression" compatibility="5.2.008" expanded="true" height="76" name="Performance" width="90" x="179" y="30">
                    <parameter key="absolute_error" value="true"/>
                    <parameter key="relative_error" value="true"/>
                    <parameter key="squared_error" value="true"/>
                    <parameter key="squared_correlation" value="true"/>
                    <parameter key="spearman_rho" value="true"/>
                    <parameter key="kendall_tau" value="true"/>
                  </operator>
                  <connect from_port="model" to_op="Apply Model" to_port="model"/>
                  <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
                  <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
                  <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
                  <portSpacing port="source_model" spacing="0"/>
                  <portSpacing port="source_test set" spacing="0"/>
                  <portSpacing port="source_through 1" spacing="0"/>
                  <portSpacing port="sink_averagable 1" spacing="0"/>
                  <portSpacing port="sink_averagable 2" spacing="0"/>
                </process>
              </operator>
              <operator activated="true" class="log" compatibility="5.2.008" expanded="true" height="76" name="Log" width="90" x="447" y="120">
                <parameter key="filename" value="C:\Users\KenanB\Desktop\Clus1Log.txt"/>
                <list key="log">
                  <parameter key="OptGnrNumber" value="operator.Optimize Parameters (Evolutionary).parameter.max_generations"/>
                  <parameter key="MAE" value="operator.Performance.value.absolute_error"/>
                  <parameter key="C" value="operator.SVM.parameter.C"/>
                  <parameter key="Gamma" value="operator.SVM.parameter.gamma"/>
                  <parameter key="Nu" value="operator.SVM.parameter.nu"/>
                  <parameter key="K" value="operator.Select by Weights.parameter.k"/>
                </list>
              </operator>
              <connect from_port="input 1" to_op="Select by Weights" to_port="weights"/>
              <connect from_port="input 2" to_op="Select by Weights" to_port="example set input"/>
              <connect from_op="Select by Weights" from_port="example set output" to_op="Validation" to_port="training"/>
              <connect from_op="Validation" from_port="model" to_port="result 1"/>
              <connect from_op="Validation" from_port="training" to_port="result 2"/>
              <connect from_op="Validation" from_port="averagable 1" to_op="Log" to_port="through 1"/>
              <connect from_op="Log" from_port="through 1" to_port="performance"/>
              <portSpacing port="source_input 1" spacing="0"/>
              <portSpacing port="source_input 2" spacing="0"/>
              <portSpacing port="source_input 3" spacing="0"/>
              <portSpacing port="sink_performance" spacing="0"/>
              <portSpacing port="sink_result 1" spacing="0"/>
              <portSpacing port="sink_result 2" spacing="0"/>
              <portSpacing port="sink_result 3" spacing="0"/>
            </process>
          </operator>
          <connect from_op="Retrieve" from_port="output" to_op="Subprocess" to_port="in 1"/>
          <connect from_op="Subprocess" from_port="out 1" to_op="Weight by Relief" to_port="example set"/>
          <connect from_op="Weight by Relief" from_port="weights" to_op="Optimize Parameters (Evolutionary)" to_port="input 1"/>
          <connect from_op="Weight by Relief" from_port="example set" to_op="Optimize Parameters (Evolutionary)" to_port="input 2"/>
          <connect from_op="Optimize Parameters (Evolutionary)" from_port="performance" to_port="result 3"/>
          <connect from_op="Optimize Parameters (Evolutionary)" from_port="parameter" to_port="result 1"/>
          <connect from_op="Optimize Parameters (Evolutionary)" from_port="result 1" to_port="result 2"/>
          <connect from_op="Optimize Parameters (Evolutionary)" from_port="result 2" to_port="result 4"/>
          <portSpacing port="source_input 1" spacing="0"/>
          <portSpacing port="sink_result 1" spacing="0"/>
          <portSpacing port="sink_result 2" spacing="0"/>
          <portSpacing port="sink_result 3" spacing="0"/>
          <portSpacing port="sink_result 4" spacing="0"/>
          <portSpacing port="sink_result 5" spacing="0"/>
        </process>
      </operator>
    </process>
  • Options
    MariusHelfMariusHelf RapidMiner Certified Expert, Member Posts: 1,869 Unicorn
    Great that it is working now. I fear that the work on the evolutionary optimization does not have a high priority. Anyway, I had a look at your process, and there is a misconfiguration in the log operator: you are trying to log the absolute error, and therefore selected the corresponding value of the *Performance* operator. However that way you will only log the value of the *last* iteration of the X-Validation. Instead, you should configure the Performance operator such that it uses the absolute error as main criterion, and log the "performance" value and maybe the "deviation" value of the X-Validation operator. That way you'll get the average performance over all iterations of the validation.

    Beh, written that I just saw that you are still using the Split Validation, not the Cross Validation, so in this case it does not matter which value you are using, nevertheless I submit my writings as a hint for the next time :)

    Best regards,
    Marius
Sign In or Register to comment.