LDA results (number of topics and words per topic)

ayaRizk

Hi, I am trying to extract topics from files and have set the number of topics to be extracted to 50 and the top words per topic to 20 (defaults are 10 and 5 respectively). But when I run the process I get about 90 topics (that also varies with different runs) and more than 50 words per topic. 

I am not sure what I'm doing wrong. Attaching the process here - appreciate the help!

Edit: Now I realized the source of the problem. The parent process "Optimize parameters" has option to set max number of topics and words per topic as the chosen parameters. I was setting their values in the LDA operator and forgetting the ones in the parent operator, which seems to override the LDA values.

Regards,
Aya

<?xml version="1.0" encoding="UTF-8"?><process version="9.10.011">

  <context>

    <input/>

    <output/>

    <macros/>

  </context>

  <operator activated="true" class="process" compatibility="9.10.011" expanded="true" name="Process">

    <parameter key="logverbosity" value="init"/>

    <parameter key="random_seed" value="2001"/>

    <parameter key="send_mail" value="never"/>

    <parameter key="notification_email" value=""/>

    <parameter key="process_duration_for_mail" value="30"/>

    <parameter key="encoding" value="SYSTEM"/>

    <process expanded="true">

      <operator activated="true" class="concurrency:loop_files" compatibility="9.10.011" expanded="true" height="82" name="Loop Files" width="90" x="112" y="34">

        <parameter key="directory" value="/Users/ayari88/Documents/Research/AFA/ROBOT/Transcriptions"/>

        <parameter key="filter_type" value="regex"/>

        <parameter key="filter_by_regex" value=".*\.docx$"/>

        <parameter key="recursive" value="true"/>

        <parameter key="skip_inaccessible" value="true"/>

        <parameter key="enable_macros" value="false"/>

        <parameter key="macro_for_file_name" value="file_name"/>

        <parameter key="macro_for_file_type" value="file_type"/>

        <parameter key="macro_for_folder_name" value="folder_name"/>

        <parameter key="reuse_results" value="false"/>

        <parameter key="enable_parallel_execution" value="true"/>

        <process expanded="true">

          <operator activated="true" class="operator_toolbox:read_word_files" compatibility="2.14.000" expanded="true" height="68" name="Read Office File" width="90" x="380" y="34">

            <parameter key="detect_file_type" value="false"/>

            <parameter key="file_extension" value="docx"/>

          </operator>

          <connect from_port="file object" to_op="Read Office File" to_port="file"/>

          <connect from_op="Read Office File" from_port="doc" to_port="output 1"/>

          <portSpacing port="source_file object" spacing="0"/>

          <portSpacing port="source_input 1" spacing="0"/>

          <portSpacing port="sink_output 1" spacing="0"/>

          <portSpacing port="sink_output 2" spacing="0"/>

        </process>

      </operator>

      <operator activated="true" class="loop_collection" compatibility="9.10.011" expanded="true" height="82" name="Loop Collection" width="90" x="313" y="34">

        <parameter key="set_iteration_macro" value="false"/>

        <parameter key="macro_name" value="iteration"/>

        <parameter key="macro_start_value" value="1"/>

        <parameter key="unfold" value="false"/>

        <process expanded="true">

          <operator activated="true" class="text:tokenize" compatibility="9.4.000" expanded="true" height="68" name="Tokenize" width="90" x="112" y="34">

            <parameter key="mode" value="non letters"/>

            <parameter key="characters" value=".:"/>

            <parameter key="language" value="English"/>

            <parameter key="max_token_length" value="3"/>

          </operator>

          <operator activated="true" class="text:transform_cases" compatibility="9.4.000" expanded="true" height="68" name="Transform Cases" width="90" x="246" y="34">

            <parameter key="transform_to" value="lower case"/>

          </operator>

          <operator activated="false" class="text:documents_to_data" compatibility="9.4.000" expanded="true" height="68" name="Documents to Data" width="90" x="313" y="442">

            <parameter key="add_meta_information" value="true"/>

            <parameter key="datamanagement" value="double_sparse_array"/>

            <parameter key="data_management" value="auto"/>

            <parameter key="use_processed_text" value="false"/>

          </operator>

          <operator activated="false" class="text:json_to_data" compatibility="9.4.000" expanded="true" height="68" name="JSON To Data" width="90" x="179" y="442">

            <parameter key="ignore_arrays" value="false"/>

            <parameter key="limit_attributes" value="false"/>

            <parameter key="skip_invalid_documents" value="false"/>

            <parameter key="guess_data_types" value="true"/>

            <parameter key="keep_missing_attributes" value="false"/>

            <parameter key="missing_values_aliases" value=", null, NaN, missing"/>

          </operator>

          <operator activated="false" class="read_csv" compatibility="9.10.011" expanded="true" height="68" name="Read CSV" width="90" x="447" y="442">

            <parameter key="column_separators" value=";"/>

            <parameter key="trim_lines" value="false"/>

            <parameter key="use_quotes" value="true"/>

            <parameter key="quotes_character" value="&quot;"/>

            <parameter key="escape_character" value="\"/>

            <parameter key="skip_comments" value="false"/>

            <parameter key="comment_characters" value="#"/>

            <parameter key="starting_row" value="1"/>

            <parameter key="parse_numbers" value="true"/>

            <parameter key="decimal_character" value="."/>

            <parameter key="grouped_digits" value="false"/>

            <parameter key="grouping_character" value=","/>

            <parameter key="infinity_representation" value=""/>

            <parameter key="date_format" value=""/>

            <parameter key="first_row_as_names" value="true"/>

            <list key="annotations"/>

            <parameter key="time_zone" value="SYSTEM"/>

            <parameter key="locale" value="English (United States)"/>

            <parameter key="encoding" value="SYSTEM"/>

            <parameter key="read_all_values_as_polynominal" value="false"/>

            <list key="data_set_meta_data_information"/>

            <parameter key="read_not_matching_values_as_missings" value="true"/>

          </operator>

          <operator activated="false" class="retrieve" compatibility="9.10.011" expanded="true" height="68" name="Retrieve" width="90" x="581" y="442">

            <parameter key="repository_entry" value="data/ROBOT/sv_stopwords_114"/>

          </operator>

          <operator activated="true" class="open_file" compatibility="9.10.011" expanded="true" height="68" name="Open File" width="90" x="246" y="136">

            <parameter key="resource_type" value="file"/>

            <parameter key="filename" value="/Users/ayari88/Documents/Research/AFA/ROBOT/RapidMiner/sv_stopwords_418.csv"/>

          </operator>

          <operator activated="true" class="text:filter_stopwords_dictionary" compatibility="9.4.000" expanded="true" height="82" name="Filter Stopwords (Dictionary)" width="90" x="380" y="34">

            <parameter key="case_sensitive" value="false"/>

            <parameter key="encoding" value="SYSTEM"/>

          </operator>

          <operator activated="true" class="text:stem_snowball" compatibility="9.4.000" expanded="true" height="68" name="Stem (Snowball)" width="90" x="514" y="34">

            <parameter key="language" value="Swedish"/>

          </operator>

          <connect from_port="single" to_op="Tokenize" to_port="document"/>

          <connect from_op="Tokenize" from_port="document" to_op="Transform Cases" to_port="document"/>

          <connect from_op="Transform Cases" from_port="document" to_op="Filter Stopwords (Dictionary)" to_port="document"/>

          <connect from_op="Open File" from_port="file" to_op="Filter Stopwords (Dictionary)" to_port="file"/>

          <connect from_op="Filter Stopwords (Dictionary)" from_port="document" to_op="Stem (Snowball)" to_port="document"/>

          <connect from_op="Stem (Snowball)" from_port="document" to_port="output 1"/>

          <portSpacing port="source_single" spacing="0"/>

          <portSpacing port="sink_output 1" spacing="0"/>

          <portSpacing port="sink_output 2" spacing="0"/>

        </process>

      </operator>

      <operator activated="true" class="concurrency:optimize_parameters_grid" compatibility="9.10.011" expanded="true" height="166" name="Optimize Parameters (Grid)" width="90" x="514" y="34">

        <list key="parameters">

          <parameter key="Extract Topics from Documents (LDA).number_of_topics" value="[1.0;100.0;10;linear]"/>

          <parameter key="Extract Topics from Documents (LDA).iterations" value="[1.0;100.0;10;linear]"/>

          <parameter key="Extract Topics from Documents (LDA).top_words_per_topic" value="[1.0;100.0;10;linear]"/>

        </list>

        <parameter key="error_handling" value="fail on error"/>

        <parameter key="log_performance" value="true"/>

        <parameter key="log_all_criteria" value="false"/>

        <parameter key="synchronize" value="false"/>

        <parameter key="enable_parallel_execution" value="true"/>

        <process expanded="true">

          <operator activated="true" class="operator_toolbox:lda" compatibility="2.14.000" expanded="true" height="124" name="Extract Topics from Documents (LDA)" width="90" x="179" y="34">

            <parameter key="number_of_topics" value="50"/>

            <parameter key="show_optimization_settings" value="true"/>

            <parameter key="use_alpha_heuristics" value="true"/>

            <parameter key="alpha_sum" value="0.1"/>

            <parameter key="use_beta_heuristics" value="true"/>

            <parameter key="beta" value="0.01"/>

            <parameter key="optimize_hyperparameters" value="true"/>

            <parameter key="optimize_interval_for_hyperparameters" value="10"/>

            <parameter key="iterations" value="1000"/>

            <parameter key="top_words_per_topic" value="20"/>

            <parameter key="stopword language" value="swedish"/>

            <parameter key="reproducible" value="false"/>

            <parameter key="enable_logging" value="false"/>

            <parameter key="use_local_random_seed" value="false"/>

            <parameter key="local_random_seed" value="1992"/>

            <parameter key="include_meta_data" value="true"/>

          </operator>

          <connect from_port="input 1" to_op="Extract Topics from Documents (LDA)" to_port="col"/>

          <connect from_op="Extract Topics from Documents (LDA)" from_port="exa" to_port="output 1"/>

          <connect from_op="Extract Topics from Documents (LDA)" from_port="top" to_port="output 2"/>

          <connect from_op="Extract Topics from Documents (LDA)" from_port="per" to_port="performance"/>

          <portSpacing port="source_input 1" spacing="0"/>

          <portSpacing port="source_input 2" spacing="0"/>

          <portSpacing port="sink_performance" spacing="0"/>

          <portSpacing port="sink_model" spacing="0"/>

          <portSpacing port="sink_output 1" spacing="0"/>

          <portSpacing port="sink_output 2" spacing="0"/>

          <portSpacing port="sink_output 3" spacing="0"/>

        </process>

      </operator>

      <connect from_op="Loop Files" from_port="output 1" to_op="Loop Collection" to_port="collection"/>

      <connect from_op="Loop Collection" from_port="output 1" to_op="Optimize Parameters (Grid)" to_port="input 1"/>

      <connect from_op="Optimize Parameters (Grid)" from_port="performance" to_port="result 1"/>

      <connect from_op="Optimize Parameters (Grid)" from_port="parameter set" to_port="result 2"/>

      <connect from_op="Optimize Parameters (Grid)" from_port="output 1" to_port="result 3"/>

      <connect from_op="Optimize Parameters (Grid)" from_port="output 2" to_port="result 4"/>

      <portSpacing port="source_input 1" spacing="0"/>

      <portSpacing port="sink_result 1" spacing="0"/>

      <portSpacing port="sink_result 2" spacing="0"/>

      <portSpacing port="sink_result 3" spacing="0"/>

      <portSpacing port="sink_result 4" spacing="0"/>

      <portSpacing port="sink_result 5" spacing="0"/>

    </process>

  </operator>

</process>