RapidMiner

RapidMiner

How to aggregate example sets in a CSV files

Contributor

How to aggregate example sets in a CSV files

Hello,

I am currently giving a try at rapidminer for time series prediction.
I have a setup where I make a windowed data set, and make a prediction for the next event in my time series.
I would like to save as CSV the prediction, but the CSV writer does not support appending apparently, and overwrites the file for each new prediction made. Is there a way to do this easily? I am sure there are far better processes to do something like this...

Thank you

Alexandre


<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.0">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" expanded="true" name="Process">
    <process expanded="true" height="710" width="1518">
      <operator activated="true" class="read_csv" expanded="true" height="60" name="Read CSV" width="90" x="45" y="30">
        <parameter key="file_name" value="/home/cesari/eclipse/Grid/MyStrategies/marketData/EURODOLLAR_Five_Minutes.csv"/>
        <parameter key="use_quotes" value="false"/>
        <parameter key="column_separators" value=";"/>
        <parameter key="date_format" value="/MM/dd HH:mm:ss.SSS"/>
      </operator>
      <operator activated="true" class="read_csv" expanded="true" height="60" name="Read CSV (2)" width="90" x="45" y="255">
        <parameter key="file_name" value="/home/cesari/eclipse/Grid/grid-analysis/results/Positions.csv"/>
        <parameter key="use_quotes" value="false"/>
        <parameter key="column_separators" value=";"/>
        <parameter key="date_format" value="/MM/dd HH:mm:ss.SSS"/>
      </operator>
      <operator activated="true" class="discretize_by_user_specification" expanded="true" height="94" name="Discretize" width="90" x="179" y="255">
        <parameter key="attribute_filter_type" value="single"/>
        <parameter key="attribute" value="Value"/>
        <list key="classes">
          <parameter key="short" value="-1.0"/>
          <parameter key="flat" value="0.0"/>
          <parameter key="long" value="1.0"/>
        </list>
      </operator>
      <operator activated="true" class="set_role" expanded="true" height="76" name="Set Role (2)" width="90" x="308" y="255">
        <parameter key="name" value="date"/>
        <parameter key="target_role" value="id"/>
      </operator>
      <operator activated="true" class="set_role" expanded="true" height="76" name="Set Role" width="90" x="308" y="30">
        <parameter key="name" value="Date"/>
        <parameter key="target_role" value="id"/>
      </operator>
      <operator activated="true" class="join" expanded="true" height="76" name="Join" width="90" x="442" y="30"/>
      <operator activated="true" class="set_role" expanded="true" height="76" name="Set Role (3)" width="90" x="576" y="30">
        <parameter key="name" value="Value"/>
        <parameter key="target_role" value="label"/>
      </operator>
      <operator activated="true" class="select_attributes" expanded="true" height="76" name="Select Attributes" width="90" x="718" y="32">
        <parameter key="attribute_filter_type" value="subset"/>
        <parameter key="attributes" value="date|Value|Open|Low|High|Close"/>
      </operator>
      <operator activated="true" class="multiply" expanded="true" height="76" name="Multiply" width="90" x="884" y="31"/>
      <operator activated="true" class="series:windowing" expanded="true" height="76" name="Windowing" width="90" x="1050" y="30">
        <parameter key="horizon" value="1"/>
        <parameter key="window_size" value="200"/>
        <parameter key="create_label" value="true"/>
        <parameter key="label_attribute" value="Value"/>
      </operator>
      <operator activated="true" class="series:sliding_window_validation" expanded="true" height="112" name="Validation" width="90" x="1251" y="30">
        <parameter key="training_window_width" value="50"/>
        <parameter key="test_window_width" value="1"/>
        <process expanded="true" height="673" width="419">
          <operator activated="true" class="support_vector_machine_libsvm" expanded="true" height="76" name="SVM" width="90" x="179" y="30">
            <parameter key="gamma" value="0.2"/>
            <parameter key="C" value="0.5"/>
            <parameter key="cache_size" value="200"/>
            <list key="class_weights"/>
          </operator>
          <connect from_port="training" to_op="SVM" to_port="training set"/>
          <connect from_op="SVM" from_port="model" to_port="model"/>
          <portSpacing port="source_training" spacing="0"/>
          <portSpacing port="sink_model" spacing="0"/>
          <portSpacing port="sink_through 1" spacing="0"/>
        </process>
        <process expanded="true" height="673" width="419">
          <operator activated="true" class="apply_model" expanded="true" height="76" name="Apply Model" width="90" x="45" y="30">
            <list key="application_parameters"/>
          </operator>
          <operator activated="true" class="multiply" expanded="true" height="94" name="Multiply (2)" width="90" x="45" y="210"/>
          <operator activated="true" class="select_attributes" expanded="true" height="76" name="Select Attributes (2)" width="90" x="112" y="390">
            <parameter key="attribute_filter_type" value="subset"/>
            <parameter key="attribute" value="prediction(label)"/>
            <parameter key="attributes" value="prediction(label)"/>
          </operator>
          <operator activated="true" class="write_csv" expanded="true" height="60" name="Write CSV" width="90" x="255" y="399">
            <parameter key="csv_file" value="/home/cesari/Bureau/previsions.csv"/>
          </operator>
          <operator activated="true" class="performance" expanded="true" height="76" name="Performance" width="90" x="246" y="30"/>
          <connect from_port="model" to_op="Apply Model" to_port="model"/>
          <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
          <connect from_op="Apply Model" from_port="labelled data" to_op="Multiply (2)" to_port="input"/>
          <connect from_op="Multiply (2)" from_port="output 1" to_op="Performance" to_port="labelled data"/>
          <connect from_op="Multiply (2)" from_port="output 2" to_op="Select Attributes (2)" to_port="example set input"/>
          <connect from_op="Select Attributes (2)" from_port="example set output" to_op="Write CSV" to_port="input"/>
          <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
          <portSpacing port="source_model" spacing="0"/>
          <portSpacing port="source_test set" spacing="0"/>
          <portSpacing port="source_through 1" spacing="0"/>
          <portSpacing port="sink_averagable 1" spacing="0"/>
          <portSpacing port="sink_averagable 2" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Read CSV" from_port="output" to_op="Set Role" to_port="example set input"/>
      <connect from_op="Read CSV (2)" from_port="output" to_op="Discretize" to_port="example set input"/>
      <connect from_op="Discretize" from_port="example set output" to_op="Set Role (2)" to_port="example set input"/>
      <connect from_op="Set Role (2)" from_port="example set output" to_op="Join" to_port="right"/>
      <connect from_op="Set Role" from_port="example set output" to_op="Join" to_port="left"/>
      <connect from_op="Join" from_port="join" to_op="Set Role (3)" to_port="example set input"/>
      <connect from_op="Set Role (3)" from_port="example set output" to_op="Select Attributes" to_port="example set input"/>
      <connect from_op="Select Attributes" from_port="example set output" to_op="Multiply" to_port="input"/>
      <connect from_op="Multiply" from_port="output 1" to_op="Windowing" to_port="example set input"/>
      <connect from_op="Windowing" from_port="example set output" to_op="Validation" to_port="training"/>
      <connect from_op="Validation" from_port="averagable 1" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>
2 REPLIES
Contributor

Re: How to aggregate example sets in a CSV files

Answer to my own question:

Easiest way for me was to use Write Special Format instead of CSV export.
Apparently many operators support appending data. It's pretty bad that the CSV doesn't.

Alexadre
Elite

Re: How to aggregate example sets in a CSV files

Hi,
we are currently revising the the operators, but to be sure, you should go to bugs.rapid-i.com and insert a feature request for that, if not already present.

Greetings,
  Sebastian
Old World Computing - Establishing the Future

Check out the Jackhammer Extension for RapidMiner! Crunch more data easier and with up to 700% speed up! Available only here