RapidMiner

Highlighted
Contributor II jhiller
Contributor II

Re: Similarity of two nominal attributes

Hi,

here is a much more efficient solution. @Thomas_Ott: What do you think about the solution? Can you explain to me, why it is not possible to use the operator "Loop Examples"? I thought that this operator does that what we do with macro "setNum" and "loopNumExamples".

<?xml version="1.0" encoding="UTF-8"?><process version="7.4.000">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="7.4.000" expanded="true" name="Process">
    <parameter key="logverbosity" value="init"/>
    <parameter key="random_seed" value="2001"/>
    <parameter key="send_mail" value="never"/>
    <parameter key="notification_email" value=""/>
    <parameter key="process_duration_for_mail" value="30"/>
    <parameter key="encoding" value="SYSTEM"/>
    <process expanded="true">
      <operator activated="true" class="retrieve" compatibility="7.4.000" expanded="true" height="68" name="Retrieve" width="90" x="45" y="34">
        <parameter key="repository_entry" value="../data/temp"/>
      </operator>
      <operator activated="true" class="subprocess" compatibility="7.4.000" expanded="true" height="82" name="getDescrSimilarity" width="90" x="179" y="34">
        <process expanded="true">
          <operator activated="true" class="multiply" compatibility="7.4.000" expanded="true" height="124" name="Multiply (2)" width="90" x="45" y="34"/>
          <operator activated="true" class="select_attributes" compatibility="7.4.000" expanded="true" height="82" name="selectGp_descr" width="90" x="179" y="187">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="gp_description"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="rename" compatibility="7.4.000" expanded="true" height="82" name="renameGp_descr" width="90" x="313" y="187">
            <parameter key="old_name" value="gp_description"/>
            <parameter key="new_name" value="description"/>
            <list key="rename_additional_attributes"/>
          </operator>
          <operator activated="true" class="select_attributes" compatibility="7.4.000" expanded="true" height="82" name="selectAi_descr" width="90" x="179" y="85">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="ai_description"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="rename" compatibility="7.4.000" expanded="true" height="82" name="renameAi_descr" width="90" x="313" y="85">
            <parameter key="old_name" value="ai_description"/>
            <parameter key="new_name" value="description"/>
            <list key="rename_additional_attributes"/>
          </operator>
          <operator activated="true" class="append" compatibility="7.4.000" expanded="true" height="103" name="appendDescr" width="90" x="447" y="136">
            <parameter key="datamanagement" value="double_array"/>
            <parameter key="data_management" value="auto"/>
            <parameter key="merge_type" value="all"/>
          </operator>
          <operator activated="true" class="text:process_document_from_data" compatibility="7.4.001" expanded="true" height="82" name="tokenizeTransform" width="90" x="581" y="136">
            <parameter key="create_word_vector" value="true"/>
            <parameter key="vector_creation" value="TF-IDF"/>
            <parameter key="add_meta_information" value="true"/>
            <parameter key="keep_text" value="false"/>
            <parameter key="prune_method" value="none"/>
            <parameter key="prune_below_percent" value="3.0"/>
            <parameter key="prune_above_percent" value="30.0"/>
            <parameter key="prune_below_rank" value="0.05"/>
            <parameter key="prune_above_rank" value="0.95"/>
            <parameter key="datamanagement" value="double_sparse_array"/>
            <parameter key="select_attributes_and_weights" value="true"/>
            <list key="specify_weights">
              <parameter key="description" value="1.0"/>
            </list>
            <process expanded="true">
              <operator activated="true" class="text:tokenize" compatibility="7.4.001" expanded="true" height="68" name="Tokenize (2)" width="90" x="45" y="34">
                <parameter key="mode" value="non letters"/>
                <parameter key="characters" value=".:"/>
                <parameter key="language" value="English"/>
                <parameter key="max_token_length" value="3"/>
              </operator>
              <operator activated="true" class="text:transform_cases" compatibility="7.4.001" expanded="true" height="68" name="Transform Cases (2)" width="90" x="179" y="34">
                <parameter key="transform_to" value="lower case"/>
              </operator>
              <connect from_port="document" to_op="Tokenize (2)" to_port="document"/>
              <connect from_op="Tokenize (2)" from_port="document" to_op="Transform Cases (2)" to_port="document"/>
              <connect from_op="Transform Cases (2)" from_port="document" to_port="document 1"/>
              <portSpacing port="source_document" spacing="0"/>
              <portSpacing port="sink_document 1" spacing="0"/>
              <portSpacing port="sink_document 2" spacing="0"/>
            </process>
          </operator>
          <operator activated="true" class="extract_macro" compatibility="7.4.000" expanded="true" height="68" name="setNum" width="90" x="715" y="136">
            <parameter key="macro" value="num"/>
            <parameter key="macro_type" value="number_of_examples"/>
            <parameter key="statistics" value="average"/>
            <parameter key="attribute_name" value=""/>
            <list key="additional_macros"/>
          </operator>
          <operator activated="true" class="concurrency:loop" compatibility="7.4.000" expanded="true" height="82" name="loopNumExamples" width="90" x="849" y="136">
            <parameter key="number_of_iterations" value="%{num}"/>
            <parameter key="iteration_macro" value="iteration"/>
            <parameter key="reuse_results" value="false"/>
            <parameter key="enable_parallel_execution" value="true"/>
            <process expanded="true">
              <operator activated="true" class="extract_macro" compatibility="7.4.000" expanded="true" height="68" name="Extract Macro" width="90" x="112" y="34">
                <parameter key="macro" value="extract_id"/>
                <parameter key="macro_type" value="data_value"/>
                <parameter key="statistics" value="average"/>
                <parameter key="attribute_name" value="id"/>
                <parameter key="example_index" value="%{iteration}"/>
                <list key="additional_macros"/>
              </operator>
              <operator activated="true" class="filter_examples" compatibility="7.4.000" expanded="true" height="103" name="Filter Examples" width="90" x="246" y="34">
                <parameter key="parameter_expression" value=""/>
                <parameter key="condition_class" value="custom_filters"/>
                <parameter key="invert_filter" value="false"/>
                <list key="filters_list">
                  <parameter key="filters_entry_key" value="id.eq.%{extract_id}"/>
                </list>
                <parameter key="filters_logic_and" value="true"/>
                <parameter key="filters_check_metadata" value="true"/>
              </operator>
              <operator activated="true" class="data_to_similarity" compatibility="7.4.000" expanded="true" height="82" name="getSimilarity" width="90" x="380" y="34">
                <parameter key="measure_types" value="NumericalMeasures"/>
                <parameter key="mixed_measure" value="MixedEuclideanDistance"/>
                <parameter key="nominal_measure" value="NominalDistance"/>
                <parameter key="numerical_measure" value="CosineSimilarity"/>
                <parameter key="divergence" value="GeneralizedIDivergence"/>
                <parameter key="kernel_type" value="radial"/>
                <parameter key="kernel_gamma" value="1.0"/>
                <parameter key="kernel_sigma1" value="1.0"/>
                <parameter key="kernel_sigma2" value="0.0"/>
                <parameter key="kernel_sigma3" value="2.0"/>
                <parameter key="kernel_degree" value="3.0"/>
                <parameter key="kernel_shift" value="1.0"/>
                <parameter key="kernel_a" value="1.0"/>
                <parameter key="kernel_b" value="0.0"/>
              </operator>
              <operator activated="true" class="similarity_to_data" compatibility="7.4.000" expanded="true" height="82" name="transformSimilarity" width="90" x="514" y="34">
                <parameter key="table_type" value="long_table"/>
              </operator>
              <connect from_port="input 1" to_op="Extract Macro" to_port="example set"/>
              <connect from_op="Extract Macro" from_port="example set" to_op="Filter Examples" to_port="example set input"/>
              <connect from_op="Filter Examples" from_port="example set output" to_op="getSimilarity" to_port="example set"/>
              <connect from_op="getSimilarity" from_port="similarity" to_op="transformSimilarity" to_port="similarity"/>
              <connect from_op="getSimilarity" from_port="example set" to_op="transformSimilarity" to_port="exampleSet"/>
              <connect from_op="transformSimilarity" from_port="exampleSet" to_port="output 1"/>
              <portSpacing port="source_input 1" spacing="0"/>
              <portSpacing port="source_input 2" spacing="0"/>
              <portSpacing port="sink_output 1" spacing="0"/>
              <portSpacing port="sink_output 2" spacing="0"/>
            </process>
          </operator>
          <operator activated="true" class="append" compatibility="7.4.000" expanded="true" height="82" name="appendResults" width="90" x="983" y="136">
            <parameter key="datamanagement" value="double_array"/>
            <parameter key="data_management" value="auto"/>
            <parameter key="merge_type" value="all"/>
          </operator>
          <operator activated="true" class="remove_duplicates" compatibility="7.4.000" expanded="true" height="82" name="Remove Duplicates" width="90" x="1117" y="136">
            <parameter key="attribute_filter_type" value="all"/>
            <parameter key="attribute" value=""/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
            <parameter key="treat_missing_values_as_duplicates" value="false"/>
          </operator>
          <operator activated="true" class="set_role" compatibility="7.4.000" expanded="true" height="82" name="Set Role" width="90" x="1251" y="136">
            <parameter key="attribute_name" value="FIRST_ID"/>
            <parameter key="target_role" value="id"/>
            <list key="set_additional_roles"/>
          </operator>
          <operator activated="true" class="rename" compatibility="7.4.000" expanded="true" height="82" name="renameSIMILARITY" width="90" x="1385" y="136">
            <parameter key="old_name" value="SIMILARITY"/>
            <parameter key="new_name" value="r_sim_descr"/>
            <list key="rename_additional_attributes"/>
          </operator>
          <operator activated="true" breakpoints="after" class="select_attributes" compatibility="7.4.000" expanded="true" height="82" name="removeSecondID" width="90" x="1519" y="136">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="SECOND_ID"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="true"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="join" compatibility="7.4.000" expanded="true" height="82" name="Join" width="90" x="1653" y="34">
            <parameter key="remove_double_attributes" value="true"/>
            <parameter key="join_type" value="inner"/>
            <parameter key="use_id_attribute_as_key" value="true"/>
            <list key="key_attributes"/>
            <parameter key="keep_both_join_attributes" value="false"/>
          </operator>
          <connect from_port="in 1" to_op="Multiply (2)" to_port="input"/>
          <connect from_op="Multiply (2)" from_port="output 1" to_op="Join" to_port="left"/>
          <connect from_op="Multiply (2)" from_port="output 2" to_op="selectAi_descr" to_port="example set input"/>
          <connect from_op="Multiply (2)" from_port="output 3" to_op="selectGp_descr" to_port="example set input"/>
          <connect from_op="selectGp_descr" from_port="example set output" to_op="renameGp_descr" to_port="example set input"/>
          <connect from_op="renameGp_descr" from_port="example set output" to_op="appendDescr" to_port="example set 2"/>
          <connect from_op="selectAi_descr" from_port="example set output" to_op="renameAi_descr" to_port="example set input"/>
          <connect from_op="renameAi_descr" from_port="example set output" to_op="appendDescr" to_port="example set 1"/>
          <connect from_op="appendDescr" from_port="merged set" to_op="tokenizeTransform" to_port="example set"/>
          <connect from_op="tokenizeTransform" from_port="example set" to_op="setNum" to_port="example set"/>
          <connect from_op="setNum" from_port="example set" to_op="loopNumExamples" to_port="input 1"/>
          <connect from_op="loopNumExamples" from_port="output 1" to_op="appendResults" to_port="example set 1"/>
          <connect from_op="appendResults" from_port="merged set" to_op="Remove Duplicates" to_port="example set input"/>
          <connect from_op="Remove Duplicates" from_port="example set output" to_op="Set Role" to_port="example set input"/>
          <connect from_op="Set Role" from_port="example set output" to_op="renameSIMILARITY" to_port="example set input"/>
          <connect from_op="renameSIMILARITY" from_port="example set output" to_op="removeSecondID" to_port="example set input"/>
          <connect from_op="removeSecondID" from_port="example set output" to_op="Join" to_port="right"/>
          <connect from_op="Join" from_port="join" to_port="out 1"/>
          <portSpacing port="source_in 1" spacing="0"/>
          <portSpacing port="source_in 2" spacing="0"/>
          <portSpacing port="sink_out 1" spacing="0"/>
          <portSpacing port="sink_out 2" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Retrieve" from_port="output" to_op="getDescrSimilarity" to_port="in 1"/>
      <connect from_op="getDescrSimilarity" from_port="out 1" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>

Yours

Johannes

RM Certified Expert
RM Certified Expert

Re: Similarity of two nominal attributes

I think you could use Loop Examples but I usually defer to the generic Loop operator. 

Polls
How can RapidMiner increase participation in our new competitions?
Twitter Feed