Collect each performance of a Backward elimination

JohnQuestJohnQuest Member Posts: 15 Contributor II
edited June 2019 in Help
I am using rapid miner for my data mining research, I used backward elimination for my feature (attribute) selections. I was wondering how to set up the process in order to gather each performance for the backward elimination. For example: feature set one (A, B, C , D, E, F), performance one(…); feature set two(A, B, C, D, E), performance two(…); ….

I am currently processing a data table with 21 features and 157000 items. A brute force feature selection simply overload my computer memory. I was wonder how to find the best combination as well as plot a graph that shows which combination of features performance low, and which combination performance high.

Thanks in advance for your kindly support.
Tagged:

Answers

  • wesselwessel Member Posts: 537 Maven
    This script reads the iris dataset, splits it in a test and training set and then logs performance for each subset on the test set.

    I had the same problem, hope this helps.
    If you need further refinement let me know.
    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
    <process version="5.0">
     <context>
       <input>
         <location/>
       </input>
       <output>
         <location/>
         <location/>
         <location/>
       </output>
       <macros/>
     </context>
     <operator activated="true" class="process" expanded="true" name="Root">
       <process expanded="true" height="638" width="573">
         <operator activated="true" class="retrieve" expanded="true" height="60" name="Retrieve" width="90" x="45" y="30">
           <parameter key="repository_entry" value="//Samples/data/Iris"/>
         </operator>
         <operator activated="true" class="rename_by_replacing" expanded="true" height="76" name="rename" width="90" x="45" y="120">
           <parameter key="replace_what" value="attribute_"/>
           <parameter key="replace_by" value="a"/>
         </operator>
         <operator activated="true" class="split_data" expanded="true" height="94" name="split" width="90" x="179" y="30">
           <enumeration key="partitions">
             <parameter key="ratio" value="0.5"/>
             <parameter key="ratio" value="0.5"/>
           </enumeration>
         </operator>
         <operator activated="true" class="optimize_selection" expanded="true" height="94" name="FS" width="90" x="313" y="30">
           <process expanded="true" height="638" width="573">
             <operator activated="true" class="select_attributes" expanded="true" height="76" name="no special" width="90" x="45" y="30">
               <parameter key="attribute_filter_type" value="regular_expression"/>
               <parameter key="regular_expression" value="class"/>
               <parameter key="invert_selection" value="true"/>
               <parameter key="include_special_attributes" value="true"/>
             </operator>
             <operator activated="true" class="execute_script" expanded="true" height="76" name="Execute Script" width="90" x="179" y="30">
               <parameter key="script" value="ExampleSet exampleSet = operator.getInput(ExampleSet.class);&#13;&#10;&#13;&#10;String atts = &quot;&quot;;&#13;&#10;&#10;for (Attribute attribute : exampleSet.getAttributes()) {&#10;    String name = attribute.getName();&#13;&#13;&#13;&#13;&#10;&#9;atts = atts + &quot; &quot; + name;&#13;&#10;}&#13;&#13;&#10;&#10;operator.getProcess().getMacroHandler().addMacro(&quot;atts&quot;, atts)"/>
             </operator>
             <operator activated="true" class="weka:W-J48" expanded="true" height="76" name="W-J48" width="90" x="179" y="165"/>
             <operator activated="true" class="apply_model" expanded="true" height="76" name="Applier" width="90" x="112" y="300">
               <list key="application_parameters"/>
             </operator>
             <operator activated="true" class="performance" expanded="true" height="76" name="Performance" width="90" x="246" y="300"/>
             <operator activated="true" class="provide_macro_as_log_value" expanded="true" height="76" name="Provide Macro as Log Value" width="90" x="313" y="30">
               <parameter key="macro_name" value="atts"/>
             </operator>
             <operator activated="true" class="log" expanded="true" height="94" name="MyLog" width="90" x="380" y="120">
               <list key="log">
                 <parameter key="generation" value="operator.FS.value.generation"/>
                 <parameter key="performance" value="operator.Performance.value.performance"/>
                 <parameter key="atts" value="operator.Provide Macro as Log Value.value.macro_value"/>
               </list>
             </operator>
             <connect from_port="example set" to_op="no special" to_port="example set input"/>
             <connect from_port="through 1" to_op="Applier" to_port="unlabelled data"/>
             <connect from_op="no special" from_port="example set output" to_op="Execute Script" to_port="input 1"/>
             <connect from_op="no special" from_port="original" to_op="W-J48" to_port="training set"/>
             <connect from_op="Execute Script" from_port="output 1" to_op="Provide Macro as Log Value" to_port="through 1"/>
             <connect from_op="W-J48" from_port="model" to_op="Applier" to_port="model"/>
             <connect from_op="Applier" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
             <connect from_op="Performance" from_port="performance" to_op="MyLog" to_port="through 1"/>
             <connect from_op="Provide Macro as Log Value" from_port="through 1" to_op="MyLog" to_port="through 2"/>
             <connect from_op="MyLog" from_port="through 1" to_port="performance"/>
             <portSpacing port="source_example set" spacing="0"/>
             <portSpacing port="source_through 1" spacing="0"/>
             <portSpacing port="source_through 2" spacing="0"/>
             <portSpacing port="sink_performance" spacing="0"/>
           </process>
         </operator>
         <operator activated="true" class="log_to_data" expanded="true" height="94" name="Log to Data" width="90" x="447" y="120">
           <parameter key="log_name" value="MyLog"/>
         </operator>
         <connect from_op="Retrieve" from_port="output" to_op="rename" to_port="example set input"/>
         <connect from_op="rename" from_port="example set output" to_op="split" to_port="example set"/>
         <connect from_op="split" from_port="partition 1" to_op="FS" to_port="example set in"/>
         <connect from_op="split" from_port="partition 2" to_op="FS" to_port="through 1"/>
         <connect from_op="FS" from_port="weights" to_port="result 1"/>
         <connect from_op="FS" from_port="performance" to_op="Log to Data" to_port="through 1"/>
         <connect from_op="Log to Data" from_port="exampleSet" to_port="result 2"/>
         <portSpacing port="source_input 1" spacing="0"/>
         <portSpacing port="sink_result 1" spacing="0"/>
         <portSpacing port="sink_result 2" spacing="0"/>
         <portSpacing port="sink_result 3" spacing="0"/>
       </process>
     </operator>
    </process>
  • wesselwessel Member Posts: 537 Maven
    Is this quirky script still needed?

    Or can you now log the performance of a sub process directly?
    And more importantly log the "attributes used" at each iteration of attribute selection.
  • landland RapidMiner Certified Analyst, RapidMiner Certified Expert, Member Posts: 2,531 Unicorn
    Hi,
    actually this isn't needed. Was it ever? Don't know exactly...Take a look at the following process:
    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
    <process version="5.0">
      <context>
        <input/>
        <output/>
        <macros/>
      </context>
      <operator activated="true" class="process" compatibility="5.0.8" expanded="true" name="Process">
        <process expanded="true" height="338" width="616">
          <operator activated="true" class="retrieve" compatibility="5.0.8" expanded="true" height="60" name="Retrieve" width="90" x="45" y="30">
            <parameter key="repository_entry" value="//Samples/data/Iris"/>
          </operator>
          <operator activated="true" class="optimize_selection_forward" compatibility="5.0.10" expanded="true" height="94" name="First Round" width="90" x="179" y="30">
            <parameter key="maximal_number_of_attributes" value="25"/>
            <parameter key="speculative_rounds" value="3"/>
            <process expanded="true" height="597" width="804">
              <operator activated="true" class="materialize_data" compatibility="5.0.10" expanded="true" height="76" name="Materialize Data" width="90" x="45" y="30"/>
              <operator activated="true" class="x_validation" compatibility="5.0.10" expanded="true" height="112" name="Validation" width="90" x="179" y="30">
                <process expanded="true" height="615" width="377">
                  <operator activated="true" class="normalize" compatibility="5.0.10" expanded="true" height="94" name="Normalize" width="90" x="45" y="30">
                    <parameter key="create_view" value="true"/>
                  </operator>
                  <operator activated="true" class="linear_regression" compatibility="5.0.10" expanded="true" height="94" name="Linear Regression" width="90" x="179" y="30">
                    <parameter key="feature_selection" value="none"/>
                    <parameter key="eliminate_colinear_features" value="false"/>
                  </operator>
                  <connect from_port="training" to_op="Normalize" to_port="example set input"/>
                  <connect from_op="Normalize" from_port="example set output" to_op="Linear Regression" to_port="training set"/>
                  <connect from_op="Normalize" from_port="preprocessing model" to_port="through 1"/>
                  <connect from_op="Linear Regression" from_port="model" to_port="model"/>
                  <portSpacing port="source_training" spacing="0"/>
                  <portSpacing port="sink_model" spacing="0"/>
                  <portSpacing port="sink_through 1" spacing="0"/>
                  <portSpacing port="sink_through 2" spacing="0"/>
                </process>
                <process expanded="true" height="615" width="413">
                  <operator activated="true" class="apply_model" compatibility="5.0.10" expanded="true" height="76" name="Apply Model (2)" width="90" x="66" y="104">
                    <list key="application_parameters"/>
                    <parameter key="create_view" value="true"/>
                  </operator>
                  <operator activated="true" class="apply_model" compatibility="5.0.10" expanded="true" height="76" name="Apply Model" width="90" x="179" y="30">
                    <list key="application_parameters"/>
                  </operator>
                  <operator activated="true" class="performance_regression" compatibility="5.0.10" expanded="true" height="76" name="Performance" width="90" x="313" y="30">
                    <parameter key="absolute_error" value="true"/>
                    <parameter key="squared_correlation" value="true"/>
                  </operator>
                  <connect from_port="model" to_op="Apply Model" to_port="model"/>
                  <connect from_port="test set" to_op="Apply Model (2)" to_port="unlabelled data"/>
                  <connect from_port="through 1" to_op="Apply Model (2)" to_port="model"/>
                  <connect from_op="Apply Model (2)" from_port="labelled data" to_op="Apply Model" to_port="unlabelled data"/>
                  <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
                  <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
                  <portSpacing port="source_model" spacing="0"/>
                  <portSpacing port="source_test set" spacing="0"/>
                  <portSpacing port="source_through 1" spacing="0"/>
                  <portSpacing port="source_through 2" spacing="0"/>
                  <portSpacing port="sink_averagable 1" spacing="0"/>
                  <portSpacing port="sink_averagable 2" spacing="0"/>
                </process>
              </operator>
              <operator activated="true" class="log" compatibility="5.0.10" expanded="true" height="76" name="Log" width="90" x="380" y="30">
                <list key="log">
                  <parameter key="numberOfAttributes" value="operator.First Round.value.number of attributes"/>
                  <parameter key="attributes" value="operator.First Round.value.feature_names"/>
                  <parameter key="absolute error" value="operator.Validation.value.performance2"/>
                  <parameter key="std" value="operator.Validation.value.deviation"/>
                  <parameter key="sqrd cor" value="operator.Validation.value.performance3"/>
                </list>
              </operator>
              <connect from_port="example set" to_op="Materialize Data" to_port="example set input"/>
              <connect from_op="Materialize Data" from_port="example set output" to_op="Validation" to_port="training"/>
              <connect from_op="Validation" from_port="averagable 1" to_op="Log" to_port="through 1"/>
              <connect from_op="Log" from_port="through 1" to_port="performance"/>
              <portSpacing port="source_example set" spacing="0"/>
              <portSpacing port="sink_performance" spacing="0"/>
            </process>
          </operator>
          <connect from_op="Retrieve" from_port="output" to_op="First Round" to_port="example set"/>
          <portSpacing port="source_input 1" spacing="0"/>
          <portSpacing port="sink_result 1" spacing="0"/>
        </process>
      </operator>
    </process>
    Greetings,
      Sebastian
Sign In or Register to comment.