Options
On Twitter API feed I am loosing the last digit on the tweet tweet_id using depivot.
Maven
Hi
I am doing some work on extracting data from the Twitter API, and I am having problems extracting the id for the tweet. I keep loosing the last character.
I am doing some work on extracting data from the Twitter API, and I am having problems extracting the id for the tweet. I keep loosing the last character.
<?xml version="1.0" encoding="UTF-8"?><process version="9.10.000">
<context>
<input/>
<output/>
<macros/>
</context>
<operator activated="true" class="process" compatibility="9.10.000" expanded="true" name="Process">
<parameter key="logverbosity" value="init"/>
<parameter key="random_seed" value="2001"/>
<parameter key="send_mail" value="never"/>
<parameter key="notification_email" value=""/>
<parameter key="process_duration_for_mail" value="30"/>
<parameter key="encoding" value="SYSTEM"/>
<process expanded="true">
<operator activated="true" class="text:create_document" compatibility="9.4.000" expanded="true" height="68" name="Create Document" width="90" x="45" y="187">
<parameter key="text" value="{"success":true,"total":5,"message":"Executed query successfully","searchTerms":"\"alexander forbes\"","data":[{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Mon Jun 26 06:40:42 +0000 2023","conversation_id_str":"1673219718721806336","favorite_count":1,"favorited":false,"full_text":"@AlexanderForbes @Alexander Forbes please respond to my email requests.l need my 2014 IRP5 for resolving with SARS.l have emailed you several times from last week.Please respond urgently.","in_reply_to_screen_name":"AlexanderForbes","in_reply_to_user_id_str":"59608915","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":0,"retweet_count":0,"retweeted":false,"user_id_str":"864307387913695232","id_str":"1673219718721806336"},"user":{"can_dm":true,"can_media_tag":true,"created_at":"Tue May 16 02:31:54 +0000 2017","default_profile_image":false,"description":"Child Of God COTH INT, IT, ICT Cloud Digital Agile Project Manager / ICT, IT Business 16 Years, Family & Music Promoter in the making","fast_followers_count":0,"favourites_count":6466,"followers_count":136,"friends_count":453,"has_custom_timelines":true,"is_translator":false,"listed_count":0,"location":"Terenure, Johannesburg SA","media_count":395,"name":"William Mashaba Siduna","normal_followers_count":136,"possibly_sensitive":false,"profile_image_url_https":"https://pbs.twimg.com/profile_images/1666330890815848450/B1LZlkMl_normal.jpg","screen_name":"WSiduna","statuses_count":1907,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun Jun 25 23:17:09 +0000 2023","conversation_id_str":"1673067026053574664","favorite_count":0,"favorited":false,"full_text":"@theskygenieshow @WrestleTalk_TV As long as there is a PPV event tonight, I don’t care. As long there is a Wrestling event, I’ll be excited","in_reply_to_screen_name":"theskygenieshow","in_reply_to_status_id_str":"1673103988861071362","in_reply_to_user_id_str":"992148401788669952","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":0,"retweet_count":0,"retweeted":false,"user_id_str":"2737469578","id_str":"1673108095667503104"},"user":{"can_dm":false,"can_media_tag":true,"created_at":"Fri Aug 08 18:40:42 +0000 2014","default_profile_image":false,"description":"","fast_followers_count":0,"favourites_count":309,"followers_count":44,"friends_count":269,"has_custom_timelines":false,"is_translator":false,"listed_count":0,"location":"Blackpool, England","media_count":8,"name":"Alexander Forbes","normal_followers_count":44,"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/2737469578/1434097773","profile_image_url_https":"https://pbs.twimg.com/profile_images/1429497043660288006/okKAPA5Z_normal.jpg","screen_name":"715Alexander","statuses_count":555,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun Jun 25 20:44:02 +0000 2023","conversation_id_str":"1673067026053574664","favorite_count":0,"favorited":false,"full_text":"@WrestleTalk_TV Okay. One less match on WrestleLeague but, it’ll still be interesting how the other matches go","in_reply_to_screen_name":"WrestleTalk_TV","in_reply_to_status_id_str":"1673067026053574664","in_reply_to_user_id_str":"448441609","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":1,"retweet_count":0,"retweeted":false,"user_id_str":"2737469578","id_str":"1673069562898653185"},"user":{"can_dm":false,"can_media_tag":true,"created_at":"Fri Aug 08 18:40:42 +0000 2014","default_profile_image":false,"description":"","fast_followers_count":0,"favourites_count":309,"followers_count":44,"friends_count":269,"has_custom_timelines":false,"is_translator":false,"listed_count":0,"location":"Blackpool, England","media_count":8,"name":"Alexander Forbes","normal_followers_count":44,"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/2737469578/1434097773","profile_image_url_https":"https://pbs.twimg.com/profile_images/1429497043660288006/okKAPA5Z_normal.jpg","screen_name":"715Alexander","statuses_count":555,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun Jun 25 14:04:09 +0000 2023","conversation_id_str":"1672961862428704768","favorite_count":0,"favorited":false,"full_text":"@Wezi_heartsound Link I join some","in_reply_to_screen_name":"Wezi_heartsound","in_reply_to_status_id_str":"1672961862428704768","in_reply_to_user_id_str":"1354300669","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":0,"retweet_count":0,"retweeted":false,"user_id_str":"1256642933744336902","id_str":"1672968926748082182"},"user":{"can_dm":false,"can_media_tag":false,"created_at":"Sat May 02 17:53:32 +0000 2020","default_profile_image":false,"description":"i am who I am what I do defines me,full time manchester united supporter, movie lover, children lover","fast_followers_count":0,"favourites_count":21069,"followers_count":556,"friends_count":2474,"has_custom_timelines":true,"is_translator":false,"listed_count":3,"location":"Lusaka, Zambia","media_count":13,"name":"Alexander Forbes������������������������������","normal_followers_count":556,"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/1256642933744336902/1588443390","profile_image_url_https":"https://pbs.twimg.com/profile_images/1509929001883738116/jMYKzyBN_normal.jpg","screen_name":"Alexand90845096","statuses_count":1580,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun Jun 25 14:02:45 +0000 2023","conversation_id_str":"1672840965927608320","favorite_count":0,"favorited":false,"full_text":"@Nachi_Mushy @glasses__girl Ok boi but mona apo it was suppose to the man is evil not men are evil","in_reply_to_screen_name":"Nachi_Mushy","in_reply_to_status_id_str":"1672860103366565888","in_reply_to_user_id_str":"3239377665","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":0,"retweet_count":0,"retweeted":false,"user_id_str":"1256642933744336902","id_str":"1672968578021175301"},"user":{"can_dm":false,"can_media_tag":false,"created_at":"Sat May 02 17:53:32 +0000 2020","default_profile_image":false,"description":"i am who I am what I do defines me,full time manchester united supporter, movie lover, children lover","fast_followers_count":0,"favourites_count":21069,"followers_count":556,"friends_count":2474,"has_custom_timelines":true,"is_translator":false,"listed_count":3,"location":"Lusaka, Zambia","media_count":13,"name":"Alexander Forbes������������������������������","normal_followers_count":556,"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/1256642933744336902/1588443390","profile_image_url_https":"https://pbs.twimg.com/profile_images/1509929001883738116/jMYKzyBN_normal.jpg","screen_name":"Alexand90845096","statuses_count":1580,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}}]}"/>
<parameter key="add label" value="false"/>
<parameter key="label_type" value="nominal"/>
</operator>
<operator activated="true" class="text:json_to_data" compatibility="9.4.000" expanded="true" height="82" name="JSON To Data" width="90" x="179" y="187">
<parameter key="ignore_arrays" value="false"/>
<parameter key="limit_attributes" value="false"/>
<parameter key="skip_invalid_documents" value="false"/>
<parameter key="guess_data_types" value="true"/>
<parameter key="keep_missing_attributes" value="false"/>
<parameter key="missing_values_aliases" value=", null, NaN, missing"/>
</operator>
<operator activated="true" class="multiply" compatibility="9.10.000" expanded="true" height="124" name="Multiply" width="90" x="313" y="187"/>
<operator activated="true" class="de_pivot" compatibility="9.10.000" expanded="true" height="82" name="De-Pivot" width="90" x="447" y="442">
<list key="attribute_name">
<parameter key="id" value="data\[\d*\].tweet.conversation_id_str"/>
<parameter key="favorite_count" value="data\[\d*\].tweet.favorite_count"/>
</list>
<parameter key="index_attribute" value="uuid"/>
<parameter key="create_nominal_index" value="false"/>
<parameter key="keep_missings" value="true"/>
</operator>
<operator activated="true" class="select_attributes" compatibility="9.10.000" expanded="true" height="82" name="Select Attributes (6)" width="90" x="581" y="442">
<parameter key="attribute_filter_type" value="regular_expression"/>
<parameter key="attribute" value=""/>
<parameter key="attributes" value=""/>
<parameter key="regular_expression" value=".*\[\d*\].*"/>
<parameter key="use_except_expression" value="false"/>
<parameter key="value_type" value="attribute_value"/>
<parameter key="use_value_type_exception" value="false"/>
<parameter key="except_value_type" value="time"/>
<parameter key="block_type" value="attribute_block"/>
<parameter key="use_block_type_exception" value="false"/>
<parameter key="except_block_type" value="value_matrix_row_start"/>
<parameter key="invert_selection" value="true"/>
<parameter key="include_special_attributes" value="false"/>
</operator>
<operator activated="true" class="select_attributes" compatibility="9.10.000" expanded="true" height="82" name="Select Attributes (23)" width="90" x="715" y="442">
<parameter key="attribute_filter_type" value="subset"/>
<parameter key="attribute" value="uuid"/>
<parameter key="attributes" value="|favorite_count|id|uuid"/>
<parameter key="use_except_expression" value="false"/>
<parameter key="value_type" value="attribute_value"/>
<parameter key="use_value_type_exception" value="false"/>
<parameter key="except_value_type" value="time"/>
<parameter key="block_type" value="attribute_block"/>
<parameter key="use_block_type_exception" value="false"/>
<parameter key="except_block_type" value="value_matrix_row_start"/>
<parameter key="invert_selection" value="false"/>
<parameter key="include_special_attributes" value="false"/>
</operator>
<operator activated="true" class="de_pivot" compatibility="9.10.000" expanded="true" height="82" name="De-Pivot (7)" width="90" x="447" y="289">
<list key="attribute_name">
<parameter key="followers_count" value="data\[\d*\].user.normal_followers_count"/>
<parameter key="id" value="data\[\d*\].tweet.conversation_id_str"/>
</list>
<parameter key="index_attribute" value="uuid"/>
<parameter key="create_nominal_index" value="false"/>
<parameter key="keep_missings" value="true"/>
</operator>
<operator activated="true" class="select_attributes" compatibility="9.10.000" expanded="true" height="82" name="Select Attributes (7)" width="90" x="581" y="289">
<parameter key="attribute_filter_type" value="regular_expression"/>
<parameter key="attribute" value=""/>
<parameter key="attributes" value=""/>
<parameter key="regular_expression" value=".*\[\d*\].*"/>
<parameter key="use_except_expression" value="false"/>
<parameter key="value_type" value="attribute_value"/>
<parameter key="use_value_type_exception" value="false"/>
<parameter key="except_value_type" value="time"/>
<parameter key="block_type" value="attribute_block"/>
<parameter key="use_block_type_exception" value="false"/>
<parameter key="except_block_type" value="value_matrix_row_start"/>
<parameter key="invert_selection" value="true"/>
<parameter key="include_special_attributes" value="false"/>
</operator>
<operator activated="true" class="select_attributes" compatibility="9.10.000" expanded="true" height="82" name="Select Attributes (24)" width="90" x="715" y="289">
<parameter key="attribute_filter_type" value="subset"/>
<parameter key="attribute" value="uuid"/>
<parameter key="attributes" value="|followers_count|id|uuid"/>
<parameter key="use_except_expression" value="false"/>
<parameter key="value_type" value="attribute_value"/>
<parameter key="use_value_type_exception" value="false"/>
<parameter key="except_value_type" value="time"/>
<parameter key="block_type" value="attribute_block"/>
<parameter key="use_block_type_exception" value="false"/>
<parameter key="except_block_type" value="value_matrix_row_start"/>
<parameter key="invert_selection" value="false"/>
<parameter key="include_special_attributes" value="false"/>
</operator>
<operator activated="true" class="de_pivot" compatibility="9.10.000" expanded="true" height="82" name="De-Pivot (8)" width="90" x="447" y="187">
<list key="attribute_name">
<parameter key="created_at" value="data\[\d*\].tweet.created_at"/>
<parameter key="screen_name" value="data\[\d*\].user.screen_name"/>
<parameter key="user_id" value="data\[\d*\].tweet.user_id_str"/>
<parameter key="full_text" value="data\[\d*\].tweet.full_text"/>
<parameter key="id" value="data\[\d*\].tweet.conversation_id_str"/>
<parameter key="id_str" value="data\[\d*\].tweet.id_str"/>
<parameter key="lang" value="data\[\d*\].tweet.lang"/>
<parameter key="quote_count" value="data\[\d*\].tweet.quote_count"/>
<parameter key="reply_count" value="data\[\d*\].tweet.reply_count"/>
<parameter key="retweet_count" value="data\[\d*\].tweet.retweet_count"/>
<parameter key="retweeted" value="data\[\d*\].tweet.retweeted"/>
</list>
<parameter key="index_attribute" value="uuid"/>
<parameter key="create_nominal_index" value="false"/>
<parameter key="keep_missings" value="true"/>
</operator>
<operator activated="true" class="select_attributes" compatibility="9.10.000" expanded="true" height="82" name="Select Attributes (17)" width="90" x="581" y="187">
<parameter key="attribute_filter_type" value="regular_expression"/>
<parameter key="attribute" value=""/>
<parameter key="attributes" value=""/>
<parameter key="regular_expression" value=".*\[\d*\].*"/>
<parameter key="use_except_expression" value="false"/>
<parameter key="value_type" value="attribute_value"/>
<parameter key="use_value_type_exception" value="false"/>
<parameter key="except_value_type" value="time"/>
<parameter key="block_type" value="attribute_block"/>
<parameter key="use_block_type_exception" value="false"/>
<parameter key="except_block_type" value="value_matrix_row_start"/>
<parameter key="invert_selection" value="true"/>
<parameter key="include_special_attributes" value="false"/>
</operator>
<operator activated="true" class="operator_toolbox:extract_sentiment" compatibility="2.11.000" expanded="true" height="103" name="Extract Sentiment" width="90" x="715" y="187">
<parameter key="model" value="vader"/>
<parameter key="text_attribute" value="full_text"/>
<parameter key="show_advanced_output" value="true"/>
<parameter key="use_default_tokenization_regex" value="true"/>
<list key="additional_words"/>
</operator>
<operator activated="true" class="operator_toolbox:set_metadata" compatibility="2.11.000" expanded="true" height="68" name="Set Meta Data (4)" width="90" x="849" y="187">
<list key="attributes">
<parameter key="id" value="numeric.regular"/>
<parameter key="id_str" value="polynominal.regular"/>
</list>
<parameter key="remove_duplicate_roles" value="false"/>
</operator>
<operator activated="true" class="rename" compatibility="9.10.000" expanded="true" height="82" name="Rename" width="90" x="983" y="187">
<parameter key="old_name" value="created_at"/>
<parameter key="new_name" value="created_at2"/>
<list key="rename_additional_attributes"/>
</operator>
<operator activated="true" class="generate_attributes" compatibility="9.10.000" expanded="true" height="82" name="Generate Attributes (3)" width="90" x="1117" y="187">
<list key="function_descriptions">
<parameter key="date" value="date_now()"/>
<parameter key="url" value="concat(concat("https://twitter.com/",screen_name),concat("/status/",str(id)))"/>
</list>
<parameter key="keep_all" value="true"/>
<description align="center" color="transparent" colored="false" width="126">Inlcude the search criteria to be written to DB</description>
</operator>
<operator activated="true" class="concurrency:join" compatibility="9.8.001" expanded="true" height="82" name="Join" width="90" x="1251" y="187">
<parameter key="remove_double_attributes" value="true"/>
<parameter key="join_type" value="inner"/>
<parameter key="use_id_attribute_as_key" value="false"/>
<list key="key_attributes">
<parameter key="id" value="id"/>
</list>
<parameter key="keep_both_join_attributes" value="false"/>
</operator>
<operator activated="true" class="concurrency:join" compatibility="9.8.001" expanded="true" height="82" name="Join (5)" width="90" x="1385" y="187">
<parameter key="remove_double_attributes" value="true"/>
<parameter key="join_type" value="inner"/>
<parameter key="use_id_attribute_as_key" value="false"/>
<list key="key_attributes">
<parameter key="id" value="id"/>
</list>
<parameter key="keep_both_join_attributes" value="false"/>
</operator>
<connect from_op="Create Document" from_port="output" to_op="JSON To Data" to_port="documents 1"/>
<connect from_op="JSON To Data" from_port="example set" to_op="Multiply" to_port="input"/>
<connect from_op="Multiply" from_port="output 1" to_op="De-Pivot (8)" to_port="example set input"/>
<connect from_op="Multiply" from_port="output 2" to_op="De-Pivot (7)" to_port="example set input"/>
<connect from_op="Multiply" from_port="output 3" to_op="De-Pivot" to_port="example set input"/>
<connect from_op="De-Pivot" from_port="example set output" to_op="Select Attributes (6)" to_port="example set input"/>
<connect from_op="Select Attributes (6)" from_port="example set output" to_op="Select Attributes (23)" to_port="example set input"/>
<connect from_op="Select Attributes (23)" from_port="example set output" to_op="Join (5)" to_port="right"/>
<connect from_op="De-Pivot (7)" from_port="example set output" to_op="Select Attributes (7)" to_port="example set input"/>
<connect from_op="Select Attributes (7)" from_port="example set output" to_op="Select Attributes (24)" to_port="example set input"/>
<connect from_op="Select Attributes (24)" from_port="example set output" to_op="Join" to_port="right"/>
<connect from_op="De-Pivot (8)" from_port="example set output" to_op="Select Attributes (17)" to_port="example set input"/>
<connect from_op="Select Attributes (17)" from_port="example set output" to_op="Extract Sentiment" to_port="exa"/>
<connect from_op="Extract Sentiment" from_port="exa" to_op="Set Meta Data (4)" to_port="exa"/>
<connect from_op="Set Meta Data (4)" from_port="exa" to_op="Rename" to_port="example set input"/>
<connect from_op="Rename" from_port="example set output" to_op="Generate Attributes (3)" to_port="example set input"/>
<connect from_op="Generate Attributes (3)" from_port="example set output" to_op="Join" to_port="left"/>
<connect from_op="Join" from_port="join" to_op="Join (5)" to_port="left"/>
<connect from_op="Join (5)" from_port="join" to_port="result 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_result 1" spacing="0"/>
<portSpacing port="sink_result 2" spacing="0"/>
</process>
</operator>
</process>
When de nesting the data I am loosing the last digit on the tweet_id, I have solved this issue before but for the life of mme cannot remember how to prevent the last character on the tweet_id being turned to a zero. HELP!
Tagged:
0
![https://pbs.twimg.com/profile_images/1666330890815848450/B1LZlkMl_normal.jpg","screen_name":"WSiduna","statuses_count":1907,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun](https://pbs.twimg.com/profile_images/1666330890815848450/B1LZlkMl_normal.jpg","screen_name":"WSiduna","statuses_count":1907,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun){kind=link}
![https://pbs.twimg.com/profile_banners/2737469578/1434097773","profile_image_url_https":"https://pbs.twimg.com/profile_images/1429497043660288006/okKAPA5Z_normal.jpg","screen_name":"715Alexander","statuses_count":555,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun](https://pbs.twimg.com/profile_banners/2737469578/1434097773","profile_image_url_https":"https://pbs.twimg.com/profile_images/1429497043660288006/okKAPA5Z_normal.jpg","screen_name":"715Alexander","statuses_count":555,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun){kind=link}
![https://pbs.twimg.com/profile_banners/1256642933744336902/1588443390","profile_image_url_https":"https://pbs.twimg.com/profile_images/1509929001883738116/jMYKzyBN_normal.jpg","screen_name":"Alexand90845096","statuses_count":1580,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun](https://pbs.twimg.com/profile_banners/1256642933744336902/1588443390","profile_image_url_https":"https://pbs.twimg.com/profile_images/1509929001883738116/jMYKzyBN_normal.jpg","screen_name":"Alexand90845096","statuses_count":1580,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}},{"tweet":{"bookmark_count":0,"bookmarked":false,"created_at":"Sun){kind=link}
![https://pbs.twimg.com/profile_banners/1256642933744336902/1588443390","profile_image_url_https":"https://pbs.twimg.com/profile_images/1509929001883738116/jMYKzyBN_normal.jpg","screen_name":"Alexand90845096","statuses_count":1580,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}}]}](https://pbs.twimg.com/profile_banners/1256642933744336902/1588443390","profile_image_url_https":"https://pbs.twimg.com/profile_images/1509929001883738116/jMYKzyBN_normal.jpg","screen_name":"Alexand90845096","statuses_count":1580,"translator_type":"none","verified":false,"want_retweets":false,"withheld_in_countries":[]}}]}){kind=link}