Options
Out of Memory Error
Contributor II
Hello,
I am trying to do sentiment analysis for whic i am trying to train the model using SVM (Linear). i have 12K examples (6K pos and 6K neg).
But when i am running the process it is failing due to out of memory error. I have 4GB of RAm in my pc.
Is it not sufficient to run this type process with my system's config?
Here is the code below:
(Ive intentionally kept the pruning parameter to .015...as it is my requirement)
I am trying to do sentiment analysis for whic i am trying to train the model using SVM (Linear). i have 12K examples (6K pos and 6K neg).
But when i am running the process it is failing due to out of memory error. I have 4GB of RAm in my pc.
Is it not sufficient to run this type process with my system's config?
Here is the code below:
(Ive intentionally kept the pruning parameter to .015...as it is my requirement)
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.2.008">
<context>
<input/>
<output/>
<macros/>
</context>
<operator activated="true" class="process" compatibility="5.2.008" expanded="true" name="Process">
<parameter key="logverbosity" value="off"/>
<process expanded="true" height="550" width="567">
<operator activated="true" class="text:process_document_from_file" compatibility="5.3.000" expanded="true" height="76" name="Process Documents from Files" width="90" x="45" y="120">
<list key="text_directories">
<parameter key="Negative" value="D:\InfoSpace\Rapid Miner\My Tokens\Negative_Movie_Reviews_6000_Samples"/>
<parameter key="Positive" value="D:\InfoSpace\Rapid Miner\My Tokens\Positive_Movie_Reviews_6000_Samples"/>
</list>
<parameter key="use_file_extension_as_type" value="false"/>
<parameter key="prune_method" value="percentual"/>
<parameter key="prunde_below_percent" value="0.015"/>
<parameter key="prune_above_percent" value="90.0"/>
<parameter key="prune_above_rank" value="0.05"/>
<process expanded="true" height="550" width="748">
<operator activated="true" class="text:tokenize" compatibility="5.3.000" expanded="true" height="60" name="Tokenize" width="90" x="45" y="75">
<parameter key="mode" value="specify characters"/>
<parameter key="characters" value=".,:;@#!? <>|\/[]()%^&-_=+""/>
<parameter key="expression" value="(and)"/>
</operator>
<operator activated="true" class="text:transform_cases" compatibility="5.3.000" expanded="true" height="60" name="Transform Cases" width="90" x="179" y="75">
<parameter key="transform_to" value="upper case"/>
</operator>
<operator activated="true" class="text:filter_stopwords_english" compatibility="5.3.000" expanded="true" height="60" name="Filter Stopwords (English)" width="90" x="313" y="75"/>
<operator activated="false" class="text:filter_tokens_by_pos" compatibility="5.3.000" expanded="true" height="60" name="Filter Tokens (2)" width="90" x="179" y="435">
<parameter key="expression" value="JJ.*|RB.*|RBR.*|RBS.*|VBG.*|NN.*"/>
</operator>
<operator activated="false" class="wordnet:open_wordnet_dictionary" compatibility="5.2.000" expanded="true" height="60" name="Open WordNet Dictionary" width="90" x="45" y="390">
<parameter key="directory" value="C:\Users\vnagpal\Downloads\dict"/>
</operator>
<operator activated="false" class="wordnet:stem_wordnet" compatibility="5.2.000" expanded="true" height="76" name="Stem (WordNet)" width="90" x="246" y="255"/>
<operator activated="true" class="text:stem_porter" compatibility="5.3.000" expanded="true" height="60" name="Stem (Porter)" width="90" x="514" y="165"/>
<operator activated="true" class="text:generate_n_grams_terms" compatibility="5.3.000" expanded="true" height="60" name="Generate n-Grams (Terms)" width="90" x="380" y="255"/>
<operator activated="true" class="text:filter_by_length" compatibility="5.3.000" expanded="true" height="60" name="Filter Tokens (by Length)" width="90" x="514" y="255">
<parameter key="min_chars" value="3"/>
</operator>
<connect from_port="document" to_op="Tokenize" to_port="document"/>
<connect from_op="Tokenize" from_port="document" to_op="Transform Cases" to_port="document"/>
<connect from_op="Transform Cases" from_port="document" to_op="Filter Stopwords (English)" to_port="document"/>
<connect from_op="Filter Stopwords (English)" from_port="document" to_op="Stem (Porter)" to_port="document"/>
<connect from_op="Stem (Porter)" from_port="document" to_op="Generate n-Grams (Terms)" to_port="document"/>
<connect from_op="Generate n-Grams (Terms)" from_port="document" to_op="Filter Tokens (by Length)" to_port="document"/>
<connect from_op="Filter Tokens (by Length)" from_port="document" to_port="document 1"/>
<portSpacing port="source_document" spacing="0"/>
<portSpacing port="sink_document 1" spacing="0"/>
<portSpacing port="sink_document 2" spacing="0"/>
</process>
</operator>
<operator activated="true" class="store" compatibility="5.2.008" expanded="true" height="60" name="Store (2)" width="90" x="246" y="300">
<parameter key="repository_entry" value="//NewLocalRepository/InfoSpace Model/WordList"/>
</operator>
<operator activated="true" class="x_validation" compatibility="5.2.008" expanded="true" height="112" name="Validation (2)" width="90" x="246" y="30">
<description>A cross-validation evaluating a decision tree model.</description>
<parameter key="number_of_validations" value="2"/>
<process expanded="true" height="497" width="232">
<operator activated="false" class="support_vector_machine" compatibility="5.2.008" expanded="true" height="112" name="SVM" width="90" x="112" y="255">
<parameter key="C" value="10.0"/>
<parameter key="convergence_epsilon" value="0.001"/>
</operator>
<operator activated="true" class="support_vector_machine_linear" compatibility="5.2.008" expanded="true" height="76" name="SVM (Linear)" width="90" x="45" y="30"/>
<connect from_port="training" to_op="SVM (Linear)" to_port="training set"/>
<connect from_op="SVM (Linear)" from_port="model" to_port="model"/>
<connect from_op="SVM (Linear)" from_port="exampleSet" to_port="through 1"/>
<portSpacing port="source_training" spacing="0"/>
<portSpacing port="sink_model" spacing="0"/>
<portSpacing port="sink_through 1" spacing="0"/>
<portSpacing port="sink_through 2" spacing="0"/>
</process>
<process expanded="true" height="497" width="299">
<operator activated="true" class="apply_model" compatibility="5.2.008" expanded="true" height="76" name="Apply Model (2)" width="90" x="45" y="30">
<list key="application_parameters"/>
</operator>
<operator activated="true" class="performance" compatibility="5.2.008" expanded="true" height="76" name="Performance (2)" width="90" x="179" y="30"/>
<connect from_port="model" to_op="Apply Model (2)" to_port="model"/>
<connect from_port="test set" to_op="Apply Model (2)" to_port="unlabelled data"/>
<connect from_op="Apply Model (2)" from_port="labelled data" to_op="Performance (2)" to_port="labelled data"/>
<connect from_op="Performance (2)" from_port="performance" to_port="averagable 1"/>
<portSpacing port="source_model" spacing="0"/>
<portSpacing port="source_test set" spacing="0"/>
<portSpacing port="source_through 1" spacing="0"/>
<portSpacing port="source_through 2" spacing="0"/>
<portSpacing port="sink_averagable 1" spacing="0"/>
<portSpacing port="sink_averagable 2" spacing="0"/>
</process>
</operator>
<operator activated="true" class="store" compatibility="5.2.008" expanded="true" height="60" name="Store" width="90" x="380" y="30">
<parameter key="repository_entry" value="//NewLocalRepository/InfoSpace Model/Model"/>
</operator>
<connect from_op="Process Documents from Files" from_port="example set" to_op="Validation (2)" to_port="training"/>
<connect from_op="Process Documents from Files" from_port="word list" to_op="Store (2)" to_port="input"/>
<connect from_op="Store (2)" from_port="through" to_port="result 2"/>
<connect from_op="Validation (2)" from_port="model" to_op="Store" to_port="input"/>
<connect from_op="Store" from_port="through" to_port="result 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_result 1" spacing="0"/>
<portSpacing port="sink_result 2" spacing="0"/>
<portSpacing port="sink_result 3" spacing="0"/>
</process>
</operator>
</process>
0
Answers
have you installed the 64 bit or the 32 bit version of RapidMiner? How much memory do you have available while running RapidMiner?
You can see by opening the System Monitor View.
Best,
Nils
Thanks for the reply.
I have a 64 bit version. I have added additional 4 GB RAM, so that makes it 8 GB RAM, still, my process takes 17 hours to complete.
System monitor shows about 5 gb usage.
Is it normal for this process to run this long in my system configuration?
Regards,
Vineet
depending on what you are doing it can take a really long time. Training a SVM is a good example for that.
Best,
Nils