Options
RanfomForest runs out of memory but WEKA version does not
Contributor I
Hi all,
I am new to using RapidMiner and I am currently training RandomForest (both versions: RM's and WEKA's) on a fairly large dataset containing ~400k instances and 8 attributes. The predicted class is binominal and I am running it on a 4CPU machine using 20G of RAM with parallelised training and testing turned on.
I have tried a test run using WEKA-RF (building 10 trees) which finished in an hour and a full run (100 trees) which has finished in 11 hours and seems fine. However, when I substitute WEKA-RF for RM-RF even the ten trees test run reports out of memory error after cca an hour. Is it possible that these classifiers have so different memory requirements or is it just me doing something badly wrong? Please find the XML of the 10tree-RM_RandomForest experiment set up attached.
Many thanks,
Anya
I am new to using RapidMiner and I am currently training RandomForest (both versions: RM's and WEKA's) on a fairly large dataset containing ~400k instances and 8 attributes. The predicted class is binominal and I am running it on a 4CPU machine using 20G of RAM with parallelised training and testing turned on.
I have tried a test run using WEKA-RF (building 10 trees) which finished in an hour and a full run (100 trees) which has finished in 11 hours and seems fine. However, when I substitute WEKA-RF for RM-RF even the ten trees test run reports out of memory error after cca an hour. Is it possible that these classifiers have so different memory requirements or is it just me doing something badly wrong? Please find the XML of the 10tree-RM_RandomForest experiment set up attached.
Any help would be greatly appreciated.
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.0">
<context>
<input/>
<output/>
<macros/>
</context>
<operator activated="true" class="process" compatibility="5.0.10" expanded="true" name="Process">
<parameter key="logverbosity" value="init"/>
<parameter key="random_seed" value="2001"/>
<parameter key="send_mail" value="never"/>
<parameter key="process_duration_for_mail" value="30"/>
<parameter key="encoding" value="SYSTEM"/>
<parameter key="parallelize_main_process" value="false"/>
<process expanded="true" height="564" width="435">
<operator activated="true" class="read_csv" compatibility="5.0.10" expanded="true" height="60" name="Read CSV" width="90" x="45" y="30">
<parameter key="file_name" value="/home/bsm/anya/tools/RapidMiner/rapidminer/repository/r9_intf50.ahpsSH.Ss.csv"/>
<parameter key="encoding" value="ISO-8859-1"/>
<parameter key="trim_lines" value="true"/>
<parameter key="skip_comments" value="true"/>
<parameter key="comment_characters" value="#"/>
<parameter key="use_first_row_as_attribute_names" value="true"/>
<parameter key="use_quotes" value="true"/>
<parameter key="quotes_character" value="""/>
<parameter key="escape_character_for_quotes" value="\"/>
<parameter key="column_separators" value=","/>
<parameter key="parse_numbers" value="true"/>
<parameter key="decimal_character" value="."/>
<parameter key="grouped_digits" value="false"/>
<parameter key="grouping_character" value=","/>
<parameter key="date_format" value="yyyy-MM-dd"/>
<parameter key="read_not_matching_values_as_missings" value="true"/>
<list key="data_set_meta_data_information">
<parameter key="0" value="patchID.true.1.id"/>
<parameter key="1" value="propensity.true.4.regular"/>
<parameter key="2" value="hydrophobicity.true.4.regular"/>
<parameter key="3" value="planarity.true.4.regular"/>
<parameter key="4" value="secondary_str.true.1.regular"/>
<parameter key="5" value="SSbonds.true.4.regular"/>
<parameter key="6" value="Hbonds.true.4.regular"/>
<parameter key="7" value="fosta_scorecons.true.4.regular"/>
<parameter key="8" value="blast_scorecons.true.4.regular"/>
<parameter key="9" value="intf_class.true.6.label"/>
</list>
<parameter key="attribute_names_already_defined" value="true"/>
</operator>
<operator activated="true" class="x_validation" compatibility="5.0.10" expanded="true" height="112" name="Validation" width="90" x="179" y="30">
<parameter key="create_complete_model" value="false"/>
<parameter key="average_performances_only" value="true"/>
<parameter key="leave_one_out" value="false"/>
<parameter key="number_of_validations" value="10"/>
<parameter key="sampling_type" value="stratified sampling"/>
<parameter key="use_local_random_seed" value="false"/>
<parameter key="local_random_seed" value="1992"/>
<parameter key="parallelize_training" value="true"/>
<parameter key="parallelize_testing" value="true"/>
<process expanded="true" height="564" width="166">
<operator activated="true" class="weka:W-RandomForest" compatibility="5.0.1" expanded="true" height="76" name="W-RandomForest" width="90" x="66" y="30">
<parameter key="I" value="10.0"/>
<parameter key="K" value="0.0"/>
<parameter key="S" value="1.0"/>
<parameter key="D" value="false"/>
</operator>
<connect from_port="training" to_op="W-RandomForest" to_port="training set"/>
<connect from_op="W-RandomForest" from_port="model" to_port="model"/>
<portSpacing port="source_training" spacing="0"/>
<portSpacing port="sink_model" spacing="0"/>
<portSpacing port="sink_through 1" spacing="0"/>
</process>
<process expanded="true" height="564" width="300">
<operator activated="true" class="apply_model" compatibility="5.0.10" expanded="true" height="76" name="Apply Model" width="90" x="45" y="30">
<list key="application_parameters"/>
<parameter key="create_view" value="false"/>
</operator>
<operator activated="true" class="performance_binominal_classification" compatibility="5.0.10" expanded="true" height="76" name="Performance" width="90" x="179" y="30">
<parameter key="main_criterion" value="first"/>
<parameter key="accuracy" value="true"/>
<parameter key="classification_error" value="false"/>
<parameter key="kappa" value="false"/>
<parameter key="AUC (optimistic)" value="false"/>
<parameter key="AUC" value="false"/>
<parameter key="AUC (pessimistic)" value="false"/>
<parameter key="precision" value="false"/>
<parameter key="recall" value="false"/>
<parameter key="lift" value="false"/>
<parameter key="fallout" value="false"/>
<parameter key="f_measure" value="false"/>
<parameter key="false_positive" value="false"/>
<parameter key="false_negative" value="false"/>
<parameter key="true_positive" value="false"/>
<parameter key="true_negative" value="false"/>
<parameter key="sensitivity" value="false"/>
<parameter key="specificity" value="false"/>
<parameter key="youden" value="false"/>
<parameter key="positive_predictive_value" value="false"/>
<parameter key="negative_predictive_value" value="false"/>
<parameter key="psep" value="false"/>
<parameter key="skip_undefined_labels" value="true"/>
<parameter key="use_example_weights" value="true"/>
</operator>
<connect from_port="model" to_op="Apply Model" to_port="model"/>
<connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
<connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
<connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
<portSpacing port="source_model" spacing="0"/>
<portSpacing port="source_test set" spacing="0"/>
<portSpacing port="source_through 1" spacing="0"/>
<portSpacing port="sink_averagable 1" spacing="0"/>
<portSpacing port="sink_averagable 2" spacing="0"/>
</process>
</operator>
<connect from_op="Read CSV" from_port="output" to_op="Validation" to_port="training"/>
<connect from_op="Validation" from_port="model" to_port="result 1"/>
<connect from_op="Validation" from_port="training" to_port="result 2"/>
<connect from_op="Validation" from_port="averagable 1" to_port="result 3"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_result 1" spacing="0"/>
<portSpacing port="sink_result 2" spacing="0"/>
<portSpacing port="sink_result 3" spacing="0"/>
<portSpacing port="sink_result 4" spacing="0"/>
</process>
</operator>
</process>
Many thanks,
Anya
0
Answers
there are two sides: The operators are implemented completely different and will hence differ in the amount of memory. Additionaly the parameters are interpreted differently and so they will simply doe something different.
Greetings,
Sebastian