<?php
require_once '../cowitter/cli_filter_application.php';

function extractPostsToDB($url, $path = null){
    require_once '..\..\sqlUtils.php';
    
    $json = file_get_contents($url);
//     $json = utf8ize($json);   
    
//     $obj = json_decode($json,true);
    $json = json_encode($json, JSON_UNESCAPED_UNICODE);
    
    $obj = json_decode($json, false, 512, JSON_UNESCAPED_UNICODE);
    $obj = json_decode($obj, true);
    
    $posts = $obj['posts'];
    $mysqli_crawler= null;
    createCrawlerConnection($mysqli_crawler);
    $i = 0;
    foreach($posts as $post){
        $post['org_text'] = $post['text'];
        $post["text"] = "< ".$post['title']." > ".$post['text'];
        $post['lang'] = $post['language'];
        $post['user_name'] = $post['author'];
        $post['created_at'] = $post['crawled'];
        $uuid = $post["uuid"];
        $post["id_str"] = $uuid;
        $post_str = json_encode($post,true);
        if(!empty($path)){
            file_put_contents($path.DIRECTORY_SEPARATOR.$uuid.".txt", $post['text']);
        }
        else{
            $value = $mysqli_crawler->real_escape_string($post_str);
            $query = "INSERT into raw_stream_data(element_id, source, data) VALUES(\"$uuid\", 3, '$value') ON DUPLICATE KEY UPDATE id=id"; //source is 3  for webhose
            $raw_parent_res = sqlQuery($mysqli_crawler, $query);
        }
     
    }
    if(!empty($obj) && $obj['requestsLeft'] == 0){
        echo "\n Out of requests left\n";
        write_to_log("Error", "Out of requests");
        return;
    }
    $next_url = "https://api.webz.io".$obj['next'];
    
    if(!empty($obj) &&  $obj['totalResults'] == 0){
        echo "\nNo results (or Finished paging results)\n";
        write_to_log("INFO", "No results (or Finished paging results)");
        return;
    }
//     extractPostsToDB("https://api.webz.io".$obj['next']);
    
}


function utf8ize($mixed) {
    if (is_array($mixed)) {
        foreach ($mixed as $key => $value) {
            $mixed[$key] = utf8ize($value);
        }
    } else if (is_string ($mixed)) {
        return utf8_encode($mixed);
    }
    return $mixed;
}
