<?php

if(!isset($_SESSION)) session_start();
require_once("demosettings.php");
require_once("parseUtils.php");
require_once('write_log.php');
require_once('searchWord.php');
require_once('sqlUtils.php');
require_once 'globalArr.php';
require_once 'sentimentUtils.php';
require_once 'analyticsUtils.php';

include_once("authorizations.php");

$action = filter_input(INPUT_GET,"Action");//setIfExists('Action',$_REQUEST);

if ($action == "asyncTendCalc") {
    $username = filter_input(INPUT_GET, "username");
    $_SESSION["username"] = $username;
    $trendName = filter_input(INPUT_GET, "trendName");
    
    write_to_log("INFO", "Creating Connection (ProSent) to $mysqli->host_info"); //in error to track the login of users   
    $mysqli = sqlCreateConnection("proSentPageEng - asyncTendCalc"); 
    $add_json_data = true;    
}

if (IS_CLI && $argv[1] === "r"){
	$username = $argv[2];
	$_SESSION["username"] = $username;
	$_SESSION["loggedIn"] = "yes";
	$trendName = $argv[3];
	$mysqli = sqlCreateConnection("proSentPageEng - CLI"); 
    $add_json_data = true;
}

include("loginCheckPoint.php");
$userdetails = getUserDetails();
$username = $userdetails['Username'];

if (array_key_exists('avoidDupTrend', $_REQUEST) && (!$_REQUEST['avoidDupTrend']) && array_key_exists("freeTextInput", $_REQUEST) && $_REQUEST["freeTextInput"]){
    $mysqli = sqlCreateConnection("proSentPageEng - freeTextInput");
    getTrend($mysqli, false);
}
else if ($add_json_data ?? false){
    $mysqli = sqlCreateConnection("proSentPageEng -add_json_data");
	getTrend($mysqli, false, $add_json_data, false, $trendName, true);
}

function is_valid_date($date){
	
	$members = explode("-",$date);
	$holder = 0;
	foreach($members as $member){
		if($member !== 'None'){
			$holder++;
		}
	}
	return ($holder >= 3);

}

function validateDate($request, $default, $connection) {
	$datePart = $_REQUEST[$request];
	
	if($datePart === "None"){
		$datePart = $default;
	}
	
	return $connection->real_escape_string($datePart);
}

function partition($list, $p) {
    $listlen = count($list);
    if ($p == 0){
        $partlen = $partrem = 0;
    }
    else {
        $partlen = floor($listlen / $p);
        $partrem = $listlen % $p;
    }
    $partition = array();
    $mark = 0;
    for($px = 0; $px < $p; $px ++) {
        $incr = ($px < $partrem) ? $partlen + 1 : $partlen;
        $partition[$px] = array_slice($list, $mark, $incr);
        $mark += $incr;
    }
    return $partition;
}

function createArrFromIndexArray($array, $index){
    $new_array = array();
    for ($i = 0; $i < sizeof($array); $i++) {
        $new_array[]=$array[$i][$index];
    }
    return $new_array;
}

function getAmountFileFormat($type, $date, $connection, $dateFormat, $savedSearchSqlId){
    global $skipDuplicate;
    
    if($type == "All"){
        if ($skipDuplicate){
            $sql_format = "SELECT SUM(count_doc) AS count FROM files,unique_files WHERE unique_files.id=files.id AND file_format IS NOT NULL AND CAST(files.$dateFormat AS DATE)='" . $date . "'" . $savedSearchSqlId;
        }
        else{
            $sql_format = "SELECT COUNT(id) AS count FROM files WHERE file_format IS NOT NULL AND CAST($dateFormat AS DATE)='" . $date . "'" . $savedSearchSqlId;
        }
    }else{
        if ($skipDuplicate){
            $sql_format = "SELECT SUM(count_doc) AS count FROM files,unique_files WHERE unique_files.id=files.id AND file_format='" . $type . "' AND CAST(files.$dateFormat AS DATE)='" . $date . "'" . $savedSearchSqlId;
        }
        else{
            $sql_format = "SELECT COUNT(id) AS count FROM files WHERE file_format='" . $type . "' AND CAST($dateFormat AS DATE)='" . $date . "'" . $savedSearchSqlId;
        }
    }
    
    $res_format = sqlQuery($connection,$sql_format);
    
    if(!empty($res_format)){
        
        $row = $res_format->fetch_object();
        
        if(!empty($row)){
            $format = $row->count;
        }
    }
    return $format;
}


function getAmountRiskByLevel($level, $date, $connection, $column, $dateFormat, $savedSearchSqlId, $isNull=false){
    if($isNull){
        $sql_risk = "SELECT COUNT(id) AS count FROM files WHERE $column IS NULL AND CAST($dateFormat AS DATE)='" . $date . "'" . $savedSearchSqlId;
    }else{
        $sql_risk = "SELECT COUNT(id) AS count FROM files WHERE $column='" . $level . "' AND CAST($dateFormat AS DATE)='" . $date . "'" . $savedSearchSqlId;
    }
    
    $res_risk = sqlQuery($connection,$sql_risk);
    
    if(!empty($res_risk)){
        
        $row = $res_risk->fetch_object();
        
        if(!empty($row)){
            $risk = $row->count;
        }
    }
    return $risk;
}

function personSentimentPerDate($sentiment, $docIds_str, $dates_range, $searchPerSql, $table, $connection, $dateFormat, $savedSearchId,$username, $isAdmin, $uniqueValuesTableName = NULL){
    $date2sent_docs = array();
    $sql_sent = sql_personSentimentPerDate($docIds_str, $dates_range, $dateFormat, $uniqueValuesTableName, $sentiment, $table, $searchPerSql, $savedSearchId, $username, $isAdmin);
    $res_sent = sqlQuery($connection,$sql_sent);
    
    while (!empty($res_sent) && ($row = $res_sent->fetch_object())){
        
        //if its total
        if(property_exists($row,"total")){
            $date2sent_docs[$row->date_col]['total'] = intval($row->total);
            if($date2sent_docs[$row->date_col]['total'] > 0){
                $date2sent_docs[$row->date_col]['occ'] = max(intval($row->occ),intval($date2sent_docs[$row->date_col]['total']));
            }
            else{
                $date2sent_docs[$row->date_col]['occ'] = 0;
            }
        }
        //2 - if its pro or anti
        else{
            $sent_captions = strtoupper($sentiment).'_caption';
            
            //if the date already in the answer array
            if(array_key_exists($row->date_col, $date2sent_docs)){
                $date2sent_docs[$row->date_col]['sum'] += intval($row->$sentiment);
                $date2sent_docs[$row->date_col]['captions'][] = $row->$sent_captions ?? "";
                $date2sent_docs[$row->date_col][$sentiment.'_count']+= intval($row->occ);
            }
            //if the date is not in the answer array
            else{
                $date2sent_docs[$row->date_col]['sum'] = intval($row->$sentiment);
                $date2sent_docs[$row->date_col]['captions'][] = arrangeCaptions($row->$sent_captions ?? []);
                $date2sent_docs[$row->date_col][$sentiment.'_count'] = intval($row->occ);
            }
        }
        
    }
    return $date2sent_docs;
}

/**
 * arrange the caption from the DB as "caption (#occ), caption1 (#occ), caption2 (#occ).."
 * @param unknown $str
 * @return string
 */
function arrangeCaptions($input){
    
    $final_res = [];
    if(is_string($input)){
        $arr = explode(',', $input);
    }
    else{
        $arr = $input;
    }
    
    $arr = array_filter($arr); // remove empty items
    foreach($arr as $element){
        $element = trim($element);
        if(!array_key_exists($element, $final_res)){
            $final_res[$element] = 0;
        }
        $final_res[$element]++;
    }
    arsort($final_res);
    $final_res = array_slice($final_res,0,10);
    $final_str = "";
    foreach($final_res as $key=>$val){
        $final_str.= "$key ($val)\n";
    }
    return trim($final_str);
}

function mood_graph_handle($dates_range, $mysqli, $isAdmin, $username, $id){
    //emotions only for refactored area!
    if(empty($id)){
        return null;
    }
    $overall_pro_count = array();
    $overall_anti_count = array();
    $overall_mood_count = array();
    // get the final dates range
    $x_label_points = $dates_range;
    //add the day after last date as a closing date (but not to the reponse xLabelPoints, as this is a copy!)
    $lastDate = end($dates_range);
    $dayAfterLastDate = date("Y-m-d", strtotime("+1 day", strtotime($lastDate)));
    $x_label_points[] = $dayAfterLastDate;
    $x_label_points_length = sizeof($x_label_points);
    // since we take the dates two at a time, we cannot take the last date because there is no date after it
    if ($x_label_points_length > 1){
        $overallSentiments = getOverallSentimentForDates($x_label_points, $mysqli, $isAdmin, $username, $id);
    }
    
    dumpDbg("overallSentiments", $overallSentiments);
    for ($i = 0; $i < $x_label_points_length - 1; $i++){
        // retrieve sentiment counts
        $startDate = $x_label_points[$i];
        if (array_key_exists($startDate, $overallSentiments)){
            $count = $overallSentiments[$startDate];
        }
        else //should that happen????
        {
            write_to_log("error", "date not found in overall sentiments: $startDate");
            continue;
            //             $count = ['pro' => 0, 'anti' => 0, 'sum' => 0];
        }
        
        // create arrays of mood counts
        $pro_array = array($startDate, $count['pro']);
        $anti_array = array($startDate, $count['anti']);
        $sum_array = array($startDate, $count['sum']);
        
        // add the data for this current date to the overall count arrays
        array_push($overall_pro_count, $pro_array);
        array_push($overall_anti_count, $anti_array);
        array_push($overall_mood_count, $sum_array);
        
    }
    return array("pro" => $overall_pro_count, "anti" => $overall_anti_count, "all" => $overall_mood_count);
    
}

function mood_graph_handle_v2($dates_range, $mysqli, $isAdmin, $username, $id){
    //emotions only for refactored area!
    if(empty($id)){
        return null;
    }
    $overall_pro_count = array();
    $overall_anti_count = array();
    $overall_mood_count = array();
    // get the final dates range
    $x_label_points = $dates_range;
    //add the day after last date as a closing date (but not to the reponse xLabelPoints, as this is a copy!)
    $lastDate = end($dates_range);
    $dayAfterLastDate = date("Y-m-d", strtotime("+1 day", strtotime($lastDate)));
    $x_label_points[] = $dayAfterLastDate;
    $x_label_points_length = sizeof($x_label_points);
    // since we take the dates two at a time, we cannot take the last date because there is no date after it
    if ($x_label_points_length > 1){
        $overallSentiments = getOverallSentimentForDates($x_label_points, $mysqli, $isAdmin, $username, $id);
    }
    
//     dumpDbg("overallSentiments", $overallSentiments);
    for ($i = 0; $i < $x_label_points_length - 1; $i++){
        // retrieve sentiment counts
        $startDate = $x_label_points[$i];
        if (array_key_exists($startDate, $overallSentiments)){
            $count = $overallSentiments[$startDate];
        }
        else
        {
            write_to_log("error", "date not found in overall sentiments: $startDate");
            continue;
        }
        
        // create arrays of mood counts
        $sum_array = array($startDate, $count['pro']-$count['anti']);
        
        // add the data for this current date to the overall count arrays
        array_push($overall_mood_count, $sum_array);
        
    }
    return array("all" => $overall_mood_count);
    
}
/**
 * How many pro/anti sentiments were found OVER ALL DOCUMENTS (whether they match the query or not).
 * note: $entityType is either anti or pro
 */
function getOverallSentimentForDates($x_label_points, $connection, $isAdmin, $username, $id){
    global $emotion_model;
    $x_label_points_length = count($x_label_points);
    $results = [];
    if ($x_label_points_length == 0){
        return $results;
    }
    
    if(!empty($id)){
        $docIds_sql ="AND files.id IN (SELECT docid from savedsearchid WHERE id=$id)";
    } else{
        $docIds_sql ="";
    }
    $startDate = $x_label_points[0];
    $endDate = $x_label_points[$x_label_points_length-1];
    $date_range = "DATE >= '$startDate 00:00' AND DATE < '$endDate 00:00'";
    
    $usersql = $isAdmin ? TRUE : " username='$username'";
    if (empty($emotion_model) || $emotion_model == 0){
        $sql_sent = "SELECT CAST(files.date AS DATE) AS docsDate, SUM(emotion.anti) AS anti,SUM(emotion.pro) AS pro,COUNT(emotion.docid) AS s, COUNT(files.id) AS sf FROM files LEFT JOIN emotion ON (emotion.anti <> 0 OR emotion.pro <> 0) AND files.id=emotion.docId WHERE  $usersql $docIds_sql AND ($date_range)";
    }else{
        $sql_sent = "SELECT CAST(files.date AS DATE) AS docsDate, SUM(emotion.anti) AS anti,SUM(emotion.pro) AS pro, SUM(emotion.e_concept) AS c,COUNT(emotion.docid) AS s FROM files,emotion WHERE files.id=emotion.docId  $docIds_sql AND  $usersql AND ($date_range)";
    }
    $sql_sent .= " GROUP BY docsDate ORDER BY docsDate ASC";
    
    $res_sent = sqlQuery($connection,$sql_sent);
    if (!$res_sent){
        return $results;
    }
    
    while($row = $res_sent->fetch_object()){
        $element = array();
        $element['anti'] = number_format(floatval($row->anti) / ($row->s*100), 2);
        $element['pro'] = number_format(floatval($row->pro) / ($row->s*100), 2);
        $element['sum'] = number_format(floatval($row->c) / ($row->s*100), 2);
        
        $results[$row->docsDate] = $element;
    }
    return $results;
    
    
    
    
    // 			$anti = $pro = $s = $t = $c = $sf = 0;
    // 			for ($x_index = 1; $x_index < $x_label_points_length; $x_index++)
        // 			{
        // 				$r = $res_sent->fetch_object();
        //     			$startSliceDate = $x_label_points[$x_index-1];
        //     			//advance to the right slice, where end index is larger that result date (but include last x label in prev slice)
    
        //     			while ($r && strtotime($r->docsDate) < strtotime($x_label_points[$x_index]))
            //     			{
            //     			    if (isset($r->pro)){
            //     			        $pro += $r->pro;
            //     			    }
            //     			    if (isset($r->anti)){
            //     			        $anti += $r->anti;
            //     			    }
            //     			    if (isset($r->s)){
            //     			        $s += $r->s;
            //     			    }
            //     			    if (isset($r->t)){
            //     			        $t += $r->t;
            //     			    }
            //     			    if (isset($r->c)){
            //     			        $c += $r->c;
            //     			    }
            //     			    if (isset($r->sf)){
            //     			        $sf += $r->sf;
            //     			    }
        
            //     				$r = $res_sent->fetch_object();
            //     			}
        
        // 			if ($s > 0){
        // 				if ($emotion_model == 1){
        // 					$sum = $t/$s;
        // 				}elseif ($emotion_model == 2){
        // 					$sum = $c/$s;
        // 				}elseif (empty($emotion_model) || $emotion_model == 0){
        // 				    if ($s > 0){
        // 						$sum = $s/$sf;
        // 				    }
        // 				    else{
        // 						$sum = 0;
        // 				    }
        // 				}
        // 			}else{
        // 				$sum = 0;
        // 			}
        
        // 			$res = [];
        // 			if ($s > 0){
        // 				$res['anti'] = ($anti/$s)*$sum;
        // 				$res['pro'] = ($pro/$s)*$sum;
        // 				$res['sum'] = $sum;
        // 			}else{
        // 				$res['anti'] = 0;
        // 				$res['pro'] = 0;
        // 				$res['sum'] = 0;
        // 			}
        // 			$results[$startSliceDate] = $res;
        // 			//initialize next iteration
        // 			if ($r)
            // 			{
            // 			    if (isset($r->pro)){
            // 			        $pro = $r->pro;
            // 			    }
            // 		        if (isset($r->anti)){
            // 		            $anti = $r->anti;
            // 		        }
            // 		        if (isset($r->s)){
            // 		            $s = $r->s;
            // 		        }
            // 		        if (isset($r->t)){
            // 		            $t = $r->t;
            // 		        }
            // 		        if (isset($r->c)){
            // 		            $c = $r->c;
            // 		        }
            // 		        if (isset($r->sf)){
            // 		            $sf = $r->sf;
            // 		        }
            // 			}
        // 			else{
        // 				break;
        // 			}
        // 	}
    // 	return $results;
}



function personSentiment($sentiment, $sent_docs){
    if ($sentiment == 'pro') {
        $pro_person = $sent_docs['sum'];
        $pro_captions = $sent_docs['captions'];
        $pro_count = $sent_docs['pro_count'];
        $entity = $sent_docs['sum'];
        $entity_local = $sent_docs['sum'];
        return  array(
            "general"=>$entity,
            "captions"=>$pro_captions,
            "captions_count"=>$pro_count,
            "local"=>$entity_local,
            "totalPro"=>$pro_person,
            "totalAnti"=>NULL,
            "proDocs"=> $sent_docs['sum']
        );
        
    }else{
        $anti_person = $sent_docs['sum'];
        $anti_captions = $sent_docs['captions'];
        $anti_count = $sent_docs['anti_count'];
        $entity = $sent_docs['sum'];
        $entity_local = $sent_docs['sum'];
        
        return array(
            "general"=>$entity,
            "captions"=>$anti_captions,
            "captions_count"=>$anti_count,
            "local"=>$entity_local,
            "totalPro"=>NULL,
            "totalAnti"=>$anti_person,
            "antiDocs"=>$sent_docs['sum']
        );
        
    }
}

function createArrayOfPoints($docs, $totalDocs, $date) {
    if($totalDocs == 0){
        $percent = 0;
    }else{
        $percent = round((($docs / $totalDocs) * 100),2);
    }
    return array($date, $percent);
}

/**
 * How many docs of the given entity were pro/anti.
 */
function entitySumSentiment($savedSearch_sql, $table, $freeTextInput, $sentiment, $connection, $dateColumn, $savedSearchSqlId, $sql_str_id) {
    $sum_docs = array();
    $sql = entity_sent_sql($savedSearch_sql, $dateColumn,$freeTextInput,$sentiment,$table,$savedSearchSqlId, $sql_str_id);
    
    $res_entity_sent_docs = sqlQuery($connection,$sql);
    
    while (!empty($res_entity_sent_docs) && ($row = $res_entity_sent_docs->fetch_object())){
        $sent_txt = strtoupper($sentiment)."_Captions";
        $sent_caption = explode(',',trim($row->$sent_txt,','));
        //if the date already exists
        if(!empty($sum_docs) && array_key_exists($row->date_col, $sum_docs)){
            $sum_docs[$row->date_col]['sum'] += $row->$sentiment;
            foreach($sent_caption as $cap){
                if(!in_array($cap, $sum_docs[$row->date_col]['captions']) && !empty($cap)){
                    $sum_docs[$row->date_col]['captions'][] = $cap;
                }
            }
            $sum_docs[$row->date_col]['count_sent'] += count($sent_caption);
        }
        else{
            //if the date not exists yet
            $sum_docs[$row->date_col]['sum'] = $row->$sentiment;
            $sum_docs[$row->date_col]['captions'] = array_unique(array_slice($sent_caption,0,10));
            $sum_docs[$row->date_col]['count_sent'] = count($sent_caption);
        }
    }
    if(empty($sum_docs)){
        $sum_docs = [];
    }
    //limit the captions to 10
    $docs_sum_keys = array_keys($sum_docs);
    for($i = 0; $i < count($docs_sum_keys); $i++){
        $key_date = $docs_sum_keys[$i];
        $sum_docs[$key_date]['captions'] = array_slice($sum_docs[$key_date]['captions'],0,10);
    }
    
    return $sum_docs;
}

function populateMatchingNames_Sentiment($mysqli,$freeTextInput,$entitiesSelection){
    $table2escapedNames = null;
    $type2entType = ["person_object" => "person,email", "place_object" => "place", "organizational_identity" => "organization"];
    $type = array_key_exists($entitiesSelection, $type2entType)? $type2entType[$entitiesSelection]:$entitiesSelection;//$type2entType[$entitiesSelection]? $type2entType[$entitiesSelection]:$entitiesSelection;
    $proConcepts = get_pro_anti_concepts('pro');
    $antiConcepts = get_pro_anti_concepts('anti');
    $conditionPro = array("sentimentType"=>"Positive","instances"=>[],'concepts'=>$proConcepts);
    $conditionAnti = array("sentimentType"=>"Negative","instances"=>[],'concepts'=>$antiConcepts);
    $namePro = populateMatchingNames($table2escapedNames, $mysqli, $freeTextInput,$type,null,$conditionPro, false);
    $nameAnti = populateMatchingNames($table2escapedNames, $mysqli, $freeTextInput,$type,null,$conditionAnti, false);
    $entityAnti = array_unique($nameAnti["Non-Aggregated"][$entitiesSelection]);
    $entityPro = array_unique($namePro["Non-Aggregated"][$entitiesSelection]);
    if(empty($entityAnti)){
        $entityAnti = $freeTextInput;
    }
    if(empty($entityPro)){
        $entityPro = $freeTextInput;
    }
    return array('entityAnti'=>$entityAnti,'entityPro'=>$entityPro);
}

//all sentiment trends except fronm person
function sentimentTrend($id, $fileName, $entitiesSelection, $freeTextInput, $isAdmin, $username, $savedSearchSqlId, $dates_range, $dateSQLFormat, $mysqli) {
    global $metadata_path;
    $array = array("organizational_identity" => 'Organizational-identity', "place_object" => 'Place-object', "abstract_idea_entity" => 'Abstract_idea_entity');
    $savedSearch_sql = "Select docId FROM savedsearchid WHERE id=$id";
    
    $tmpDate2pro = $tmpDate2anti = $tmpDate2allEntities = null;
    
    if ($entitiesSelection === "event") {
        $tempfreeTextInput = $freeTextInput;
    } else if (!empty($array[$entitiesSelection])) {
        $tempfreeTextInput = analyzeWord($freeTextInput, $array[$entitiesSelection]);
    }
    
    
    $date2allDocs = getNumofAllAnalayzedDocsPerDatesRange($dates_range, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId);
    $date2allEntities = getEntitySumAppearancesInDatesRange($savedSearch_sql, $dates_range, $freeTextInput, $entitiesSelection, $mysqli, $dateSQLFormat, $savedSearchSqlId, $isAdmin, $username);
    
    if (strcasecmp($tempfreeTextInput, $freeTextInput) != 0) {
        $tmpDate2allEntities = getEntitySumAppearancesInDatesRange($savedSearch_sql, $dates_range, $tempfreeTextInput, $entitiesSelection, $mysqli, $dateSQLFormat, $savedSearchSqlId, $isAdmin, $username);
    }
    
    //     $entity = populateMatchingNames_Sentiment($mysqli,$freeTextInput,$entitiesSelection);
    $entity = $freeTextInput;
    $sql_sentimentConcepts = sql_getEntityIdsAppearancesInDatesRangeSentiment($dates_range, $dateSQLFormat, $entitiesSelection, $entity, $isAdmin, $username, $savedSearchSqlId);
    
    //seems like a mistake - why when it's abstract idea - to allow all to be the same?
    //     $searchPerSql = sql_getEntityIdsAppearancesInDatesRange($savedSearch_sql, $dates_range, $dateSQLFormat, $entitiesSelection, $freeTextInput, $isAdmin, $username, $savedSearchSqlId);
    //     $docIds_str = "SELECT docId FROM ($searchPerSql )as docIds";
    //     if($entitiesSelection=="abstract_idea_entity"){
    //         $sql_sentimentConcepts["pro"] = $sql_sentimentConcepts["anti"] = $docIds_str;
    //     }
    
    $date2pro = entitySumSentiment($savedSearch_sql, $entitiesSelection, $freeTextInput, "pro", $mysqli, $dateSQLFormat, $savedSearchSqlId, $sql_sentimentConcepts["pro"]);
    $date2anti = entitySumSentiment($savedSearch_sql, $entitiesSelection, $freeTextInput, "anti", $mysqli, $dateSQLFormat, $savedSearchSqlId, $sql_sentimentConcepts["anti"]);
    
    $docIds_sql = sql_getSentimentsDocsIds($savedSearch_sql, $isAdmin, $username, $entitiesSelection, $dates_range, $freeTextInput, $dateSQLFormat, $savedSearchSqlId);
    
    if (!empty($tempfreeTextInput) && strcasecmp($tempfreeTextInput, $freeTextInput) != 0) {
        $tmpDate2pro = entitySumSentiment($savedSearch_sql, $entitiesSelection, $freeTextInput, "pro", $mysqli, $dateSQLFormat, $savedSearchSqlId, $docIds_sql);
        $tmpDate2anti = entitySumSentiment($savedSearch_sql, $entitiesSelection, $freeTextInput, "anti", $mysqli, $dateSQLFormat, $savedSearchSqlId, $docIds_sql);
    }
    
    //Green line - pro
    $pro  = getSentimentLineForAllDates($dates_range, $date2allDocs, $date2allEntities, $date2pro, $tempfreeTextInput, $freeTextInput, $tmpDate2pro);
    //Red line - anti
    $anti = getSentimentLineForAllDates($dates_range, $date2allDocs, $date2allEntities, $date2anti, $tempfreeTextInput, $freeTextInput, $tmpDate2anti);
    //Black line - total
    $all = getTotalLineForAllDates($dates_range, $date2allEntities, $tempfreeTextInput, $freeTextInput, $date2allDocs, $tmpDate2allEntities);
    
    $all_ent_occ = getSumForAllDates($dates_range, $date2allEntities, $date2allDocs, $date2allEntities, $tempfreeTextInput, $freeTextInput, 'total_doc', $tmpDate2allEntities = null);
    $all_ent_pro = getSumForAllDates($dates_range, $date2allEntities, $date2allDocs, $date2pro, $tempfreeTextInput, $freeTextInput, 'sum', $tmpDate2pro = null);
    $all_ent_anti = getSumForAllDates($dates_range, $date2allEntities, $date2allDocs, $date2anti, $tempfreeTextInput, $freeTextInput, 'sum', $tmpDate2anti = null);
    $all_ana_txt = array_sum($date2allDocs);
    
    dataToFile($dates_range, $metadata_path, $fileName, $date2allDocs, $date2allEntities, $all['docs'], $pro['docs'], $anti['docs']);
    
    $res =array('entity_pro_docs'=>$pro['docs'],
        'entity_pro_caption_docs'=>$pro['caption_docs'],
        'entity_pro_caption_count'=>$pro['caption_count'],
        'entity_anti_docs'=>$anti['docs'],
        'entity_anti_caption_docs'=>$anti['caption_docs'],
        'entity_anti_caption_count'=>$anti['caption_count'],
        'docs_and_dates'=>$all['docs'],
        'occ_and_dates'=>$all['occ_and_dates'],
        'total_and_dates'=>$all['total_and_dates'],
        'all_ent_occ'=>$all_ent_occ,
        'all_ent_pro'=>$all_ent_pro,
        'all_ent_anti'=>$all_ent_anti,
        'all_ana_txt'=>$all_ana_txt);
    return $res;
}

function getThreatRatingTrend($chart_max_size,$asAvg,$metadata_path,$fileName,$dates_range,$mysqli,$dateSQLFormat,$username, $isAdmin, $savedSearchSqlId,$each_array_size,$entitiesSelection){
    $dates_chunk_array = partition($dates_range, $chart_max_size);
    $no_risk_arr = $low_risk_arr = $med_risk_arr = $high_risk_arr = insertChunkDates($dates_chunk_array);
    $new_dates_range = $dates_range;
    $risk_options = array();
    $response = array();
    $all_high_docs = 0;
    $all_med_docs = 0;
    $all_low_docs = 0;
    $all_no_docs = 0;
    $holder = 0;
    $counter = 0;
    $dates_holder = 0;
    $bet_dates_sum = 0;
    $array_no_risk = array();
    $array_h_risk = array();
    $array_m_risk = array();
    $array_l_risk = array();
    $full_high_docs_num = 0;
    $full_med_docs_num = 0;
    $full_low_docs_num = 0;
    $full_no_docs_num = 0;
    $file_titles = "Date;";
    
    $isHigh =  key2val($_REQUEST, "isHigh");
    $isMedium = key2val($_REQUEST,"isMedium");
    $isLow = key2val($_REQUEST,"isLow");
    $isNoRisk = key2val($_REQUEST,"isNoRisk");
    
    if($isHigh == "true"){
        $risk_options[] = "isHigh";
        $file_titles .= "High;";
    }
    if($isMedium == "true"){
        $risk_options[] = "isMedium";
        $file_titles .= "Medium;";
    }
    if($isLow == "true"){
        $risk_options[] = "isLow";
        $file_titles .= "Low;";
    }
    if($isNoRisk == "true"){
        $risk_options[] = "isNoRisk";
        $file_titles .= "No-Risk;";
    }
    $file_titles .= "\n";
    
    file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv", "$file_titles\n");
    $date2allDocs = getNumofAllAnalayzedDocsPerDatesRange($dates_range, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId);
    
    foreach ($dates_range as $date){
        //Sum of docs that were analayzed in this date
        if (!array_key_exists($date, $date2allDocs)){
            write_to_log("error", "missing date: $date in " . print_r($date2allDocs, true));
        }
        $all_date_docs = $date2allDocs[$date];
        if($holder == $each_array_size){
            
            //Averages
            $avg_h = ($all_high_docs / $bet_dates_sum) * 100;
            $avg_m = ($all_med_docs / $bet_dates_sum) * 100;
            $avg_l = ($all_low_docs / $bet_dates_sum)*100;
            $avg_n = ($all_no_docs / $bet_dates_sum) * 100;
            
            
            //Push to arrays
            array_push($high_risk_arr[$dates_holder], $avg_h);
            array_push($med_risk_arr[$dates_holder], $avg_m);
            array_push($low_risk_arr[$dates_holder], $avg_l);
            array_push($no_risk_arr[$dates_holder], $avg_n);
            $dates_holder++;
            $holder = 0;
            $all_high_docs = 0;
            $all_med_docs = 0;
            $all_low_docs = 0;
            $all_no_docs = 0;
            $bet_dates_sum = 0;
            $counter++;
        }
        
        $file_content = $date . ";";
        
        // 			$all_ana_txt += $all_date_docs;
        $bet_dates_sum += $all_date_docs;
        
        if($isHigh == "true"){
            $high = getAmountRiskByLevel("High", $date, $mysqli, $entitiesSelection, $dateSQLFormat, $savedSearchSqlId);
            $per_high = ($high / $all_date_docs) * 100;
            array_push($array_h_risk, $per_high);
            $all_high_docs += $high;
            $full_high_docs_num +=$high;
            $file_content .= $high . ";";
        }
        if($isMedium == "true"){
            $medium = getAmountRiskByLevel("medium", $date, $mysqli, $entitiesSelection, $dateSQLFormat, $savedSearchSqlId);
            $per_med = ($medium / $all_date_docs) * 100;
            array_push($array_m_risk, $per_med);
            $all_med_docs += $medium;
            $full_med_docs_num += $medium;
            $file_content .= $medium . ";";
        }
        if($isLow == "true"){
            $low = getAmountRiskByLevel("Low", $date, $mysqli, $entitiesSelection, $dateSQLFormat, $savedSearchSqlId);
            $per_low = ($low / $all_date_docs) * 100;
            array_push($array_l_risk, $per_low);
            $all_low_docs += $low;
            $full_low_docs_num += $low;
            $file_content .= $low . ";";
        }
        if($isNoRisk == "true"){
            $noRisk = getAmountRiskByLevel(null, $date, $mysqli, $entitiesSelection, $dateSQLFormat, $savedSearchSqlId, true);
            $per_no = ($noRisk / $all_date_docs) * 100;
            array_push($array_no_risk, $per_no);
            $all_no_docs += $noRisk;
            $full_no_docs_num += $noRisk;
            $file_content .= $noRisk . ";";
        }
        
        //				Writing to file
        file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv" ,"$file_content\n",FILE_APPEND);
        
        $holder++;
        if($counter == CHART_MAX_SIZE-1){
            //						Averages
            $avg_h = ($all_high_docs / $bet_dates_sum) * 100;
            $avg_m = ($all_med_docs / $bet_dates_sum) * 100;
            $avg_l = ($all_low_docs / $bet_dates_sum)*100;
            $avg_n = ($all_no_docs / $bet_dates_sum) * 100;
            
            //						Push to arrays
            array_push($high_risk_arr[$dates_holder], $avg_h);
            array_push($med_risk_arr[$dates_holder], $avg_m);
            array_push($low_risk_arr[$dates_holder], $avg_l);
            array_push($no_risk_arr[$dates_holder], $avg_n);
        }
    }
    
    $response["fileName"] = $fileName;
    $response["risk"] = true;
    $response["first"] = $dates_range[0];
    $response["last"] = end($dates_range);
    $response["highNum"] = $full_high_docs_num;
    $response["medNum"] = $full_med_docs_num;
    $response["lowNum"] = $full_low_docs_num;
    $response["noRiskNum"] = $full_no_docs_num;
    $response["riskOptions"] = $risk_options;
    
    
    if($asAvg == "true"){
        
        if($isHigh == "true"){
            $response["high"] = createArrFromIndexArray($high_risk_arr, 1);
        }else{
            $response["high"] = array();
        }
        if($isMedium == "true"){
            $response["medium"] = createArrFromIndexArray($med_risk_arr,1);
        }else{
            $response["medium"] =	array();
        }
        if($isLow == "true"){
            $response["low"] = createArrFromIndexArray($low_risk_arr,1);
        }else{
            $response["low"] = array();
        }
        $response["noRisk"] = ($isNoRisk == "true") ? createArrFromIndexArray($no_risk_arr,1) : array();
        
        $response["xLabelPoints"] = $new_dates_range;
        $response["asAvg"] = true;
        
        
    }else{
        $response["high"] = $array_h_risk;
        $response["medium"] = $array_m_risk;
        $response["low"] = $array_l_risk;
        $response["noRisk"] = $array_no_risk;
        $response["xLabelPoints"] = $dates_range;
        $response["asAvg"] = false;
        
    }
    return $response;
}


function getSumPerDate($array, $date, $tempfreeTextInput, $freeTextInput, $type, $tmpDate2allEntities = null) {
    if (array_key_exists($date, $array)) {
        $entity_date_sum = $array[$date][$type];
        $tmp_entity_date_sum = 0;
        if (isset($tmpDate2allEntities[$date]) && strcasecmp($tempfreeTextInput, $freeTextInput) != 0) {
            $tmp_entity_date_sum = $tmpDate2allEntities[$date][$type];
        }
        $entity_date_sum += $tmp_entity_date_sum;
        return $entity_date_sum;
    }
    return 0;
}

function getSumForAllDates($dates_range,$date2allEntities,$date2allDocs,$array, $tempfreeTextInput, $freeTextInput, $type, $tmpDate2allEntities = null) {
    $sum = 0;
    foreach ($dates_range as $date) {
        if (!isValidDate($date, $date2allDocs, $date2allEntities)) {continue;}
        $sum += getSumPerDate($array, $date, $tempfreeTextInput, $freeTextInput, $type, $tmpDate2allEntities = null);
    }
    return $sum;
}

function getSentimentLineForAllDates($dates_range, $date2allDocs, $date2allEntities, $date2Sentiment, $tempfreeTextInput, $freeTextInput, $tmpDate2Sentiment) {
    $docs = array();
    $caption_docs = array();
    $caption_count = array();
    
    foreach ($dates_range as $date) {
        //         if (!isValidDate($date, $date2allDocs, $date2allEntities) || !isset($date2Sentiment[$date]) ) {
        //             continue;
        //         }
        
        $sentiment_docs = $date2Sentiment[$date];
        
        $sentimentDocsSum = getSumPerDate($date2Sentiment, $date, $tempfreeTextInput, $freeTextInput, 'sum', $tmpDate2Sentiment);
        array_push($docs, array($date, $sentimentDocsSum ?? 0));
        
        $sentiment_captions = empty($sentiment_docs['captions']) ? "" : $sentiment_docs['captions'];
        array_push($caption_docs, array($date, $sentiment_captions));
        
        $sentiment_count = empty($sentiment_docs['count_sent']) ? 0 : $sentiment_docs['count_sent'];
        array_push($caption_count, array($date, $sentiment_count));
    }
    return array('docs' => $docs, 'caption_docs' => $caption_docs, 'caption_count' => $caption_count);
}


function getTotalLineForAllDates($dates_range, $date2allEntities, $tempfreeTextInput, $freeTextInput, $date2allDocs, $tmpDate2allEntities) {
    $docs_and_dates = array();
    $occ_and_dates = array();
    $total_and_dates = array();
    
    foreach ($dates_range as $date) {
        //         if (!isValidDate($date, $date2allDocs, $date2allEntities)) {
        //             continue;
        //         }
        $entity_date_sum_docs = getSumPerDate($date2allEntities, $date, $tempfreeTextInput, $freeTextInput, 'total_doc', $tmpDate2allEntities);
        $entity_date_sum_occs = getSumPerDate($date2allEntities, $date, $tempfreeTextInput, $freeTextInput, 'total_occ', $tmpDate2allEntities);
        array_push($docs_and_dates, array($date, $entity_date_sum_docs));
        array_push($occ_and_dates, array($date, $entity_date_sum_occs));
        array_push($total_and_dates, array($date, $date2allDocs[$date]));
    }
    return array('docs' => $docs_and_dates, 'occ_and_dates' => $occ_and_dates, 'total_and_dates' => $total_and_dates);
}

function convertArray($arr) {
    $newArray = array();
    if(!$arr){
        return [];
    }
    foreach ($arr as $key) {
        $newArray["$key[0]"] = $key[1];
    }
    return $newArray;
}

//Writing to file
function dataToFile($dates_range, $metadata_path, $fileName, $date2allDocs, $date2allEntities, $entity_date_sum_docs = [], $entity_pro_docs = [], $entity_anti_docs = []) {
    $pro_docsArr = convertArray($entity_pro_docs);
    $anti_docsArr = convertArray($entity_anti_docs);
    $date_sum_docsArr = convertArray($entity_date_sum_docs);
    foreach ($dates_range as $date) {
        //         if (!isValidDate($date, $date2allDocs, $date2allEntities)) {
        //             continue;
        //         }
        $proSum = array_key_exists($date, $pro_docsArr) ? $pro_docsArr[$date] : 0;
        $antiSum = array_key_exists($date, $anti_docsArr) ? $anti_docsArr[$date] : 0;
        $date_sum_docs = array_key_exists($date, $date_sum_docsArr) ? $date_sum_docsArr[$date] : 0;
        file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv", $date . ';' . $date_sum_docs . ';' . $proSum . ';' . $antiSum . "\n", FILE_APPEND);
    }
}

function isValidDate($date, $date2allDocs, $date2allEntities) {
    if (!array_key_exists($date, $date2allDocs)) {
        write_to_log("error", "missing date: $date in " . print_r($date2allDocs, true));
    }
    if (empty($date2allEntities[$date])) {
        return false;
    }
    return true;
}

//this func take dates from the first chunk of dates_chunk_array
function insertChunkDates($dates_chunk_array) {
    $array = [];
    foreach ($dates_chunk_array as $d) {
        $bDates = array($d[0]);
        array_push($array, $bDates);
    }
    return $array;
}

function getTrend($mysqli, $v2 = false, $jsonArr = null, $print = true, $title = null, $cache = false) {
    global $metadata_path;
    global $username;
    global $dropdownArr;
    global $numberOfUsersTablesName;
    global $dropdownArrCaptionMapping, $dropdownArrMaxMapping;
    $isAdmin = getUserPermission(CAN_SHOW_ALL_USER);
    
    // Write session data and end session
    session_write_close();
    $from_dash = false;
    if ($jsonArr) {
        $from_dash = true;
        $stats =  getTableElementData($mysqli, 'dashboard', $title, 'username', $username);
        $jsonArr = json_decode($stats->details, true);
        
        //Sets the $pastPeriod variable
        $pastPeriod = getPastPeriod($mysqli, $username);
        
        if (!isset($jsonArr["pastPeriod"]) || intval($jsonArr["pastPeriod"]) < 0 || $jsonArr["pastPeriod"] == "null") {
            $jsonArr["pastPeriod"] = $pastPeriod;
        }
    }
    
    
    if ($jsonArr){
        if(array_key_exists("fromDate",$jsonArr) && array_key_exists("toDate",$jsonArr)){
            $fromDate = $jsonArr["fromDate"];
            $toDate = $jsonArr["fromDate"];
        }
        elseif (array_key_exists("pastPeriod",$jsonArr) && $jsonArr["pastPeriod"] != "null"){
            $toDate = date("Y-m-d");
            $fromDate = date("Y-m-d",strtotime("$toDate -" . $jsonArr["pastPeriod"] . " days"));
        }
        else{
            if (array_key_exists("fromDay",$jsonArr)){
                $fromDay = $jsonArr["fromDay"];
            }
            if (array_key_exists("fromMonth",$jsonArr)){
                $fromMonth = $jsonArr["fromMonth"];
            }
            if (array_key_exists("fromYear",$jsonArr)){
                $fromYear = $jsonArr["fromYear"];
            }
            if (array_key_exists("toDay",$jsonArr)){
                $toDay = $jsonArr["toDay"];
            }
            if (array_key_exists("toMonth",$jsonArr)){
                $toMonth = $jsonArr["toMonth"];
            }
            if (array_key_exists("toYear",$jsonArr)){
                $toYear = $jsonArr["toYear"];
            }
            if (array_key_exists("toYear",$jsonArr)){
                $toYear = $jsonArr["toYear"];
                $fromDate = $fromYear . "-" . $fromMonth . "-" . $fromDay;
                $toDate = $toYear . "-" . $toMonth . "-" . $toDay;
            }
        }
        
        if ($jsonArr["freeTextInput"]){
            $freeTextInput = $jsonArr["freeTextInput"];
        }
        if (array_key_exists("entitiesSelection",$jsonArr)){
            $entitiesSelection = $jsonArr["entitiesSelection"];
        }
        
        if (array_key_exists("columnNum",$jsonArr)){
            $columnNum = $jsonArr["columnNum"];
        }
        if (array_key_exists("asAvg",$jsonArr)){
            $asAvg = $jsonArr["asAvg"];
        }
        if (array_key_exists("uniqueTable",$jsonArr)){
            $uniqueTable = $jsonArr["uniqueTable"];
        }
        if (array_key_exists("savedsearch",$jsonArr)){
            $savedSearch = $jsonArr["savedsearch"];
        }
        if (array_key_exists("isTotalMentionsHidden",$jsonArr)){
            $isTotalMentionsHidden = $jsonArr["isTotalMentionsHidden"];
        }
        if (array_key_exists("isDateRangeLimitedToMentions",$jsonArr)){
            $isDateRangeLimitedToMentions = $jsonArr["isDateRangeLimitedToMentions"];
        }
        if (array_key_exists("lineGraphOptions",$jsonArr)){
            $lineGraphOptions = $jsonArr["lineGraphOptions"];
        }
        if (array_key_exists("barGraphOptions",$jsonArr)){
            $barGraphOptions = $jsonArr["barGraphOptions"];
        }
    }
    else{
//         if(!empty($v2)){
//             foreach($_REQUEST as $key=>&$element){
//                 if(in_array($element, ['true', 'false'])){
//                     $element = $element === 'true'? true: false;
//                 }
//                 if(empty($element)){
//                     write_to_log("ERROR", $key);
//                     unset($_REQUEST[$key]);
//                 }
//             }
// //             $data =  $_REQUEST["group_input"] ?? file_get_contents('php://input');
//             // $data = "{\"TemplateName\":\"proSentResults\",\"asAvg\":true,\"collapse\":true,\"columnNum\":20,\"entitiesSelection\":\"person_object\",\"freeTextInput\":\"ahmad\",\"fromDay\":14,\"fromMonth\":8,\"fromYear\":2019,\"intervalType\":\"range\",\"isDateRangeLimitedToMentions\":true,\"toDay\":29,\"toMonth\":9,\"toYear\":2019,\"topBannerDisplay\":\"none\"}";
// //             $_REQUEST = array_merge($_REQUEST,json_decode($data,1)); //merge to the request
//         }
        
        if (!empty($_REQUEST["fromDate"]) && !empty($_REQUEST["toDate"])){
            $toDate = $_REQUEST["toDate"];
            $fromDate = $_REQUEST["fromDate"];
        }
        elseif (isset($_REQUEST["pastPeriod"]) && $_REQUEST["pastPeriod"]){
            $toDate = date("Y-m-d");
            $fromDate = date("Y-m-d",strtotime("$toDate -" . $_REQUEST["pastPeriod"] . " days"));
        }
        
        if($_REQUEST["intervalType"]=="range" && $_REQUEST["startDate"] && $_REQUEST["endDate"]){
            $fromDate = new DateTime( $_REQUEST["startDate"] , new DateTimeZone('GMT'));
            $toDate = new DateTime( $_REQUEST["endDate"] , new DateTimeZone('GMT'));
            $fromDate = $fromDate->format('Y-m-d H:i:s');
            $toDate = $toDate->format('Y-m-d H:i:s');
        }
        
        $freeTextInput = key2val($_REQUEST, "freeTextInput");
        $entitiesSelection = strtolower($mysqli->real_escape_string($_REQUEST["entitiesSelection"]));
        $columnNum = key2val($_REQUEST,"columnNum");
        $asAvg = key2val($_REQUEST,"asAvg");
        $uniqueTable = key2val($_REQUEST,"uniqueTable");
        $savedSearch =  key2val($_REQUEST,"savedsearch");
        
        $isTotalMentionsHidden = key2val($_REQUEST,"isTotalMentionsHidden");
        $isDateRangeLimitedToMentions = key2val($_REQUEST,"isDateRangeLimitedToMentions");
        $lineGraphOptions = key2val($_REQUEST,"lineGraphOptions");
        $barGraphOptions = key2val($_REQUEST,"barGraphOptions");
    }
    $freeTextInputStr = $freeTextInput;
    $freeTextInput = $mysqli->real_escape_string($freeTextInput);
    
    
    //add time to date
    if(!empty($fromDate)&&!empty($toDate)){
        $fromDate .= " 00:00:00";
        $toDate .= " 23:59:59";
        $dates_range_str = $fromDate." AND ".$toDate;
    }
    else{
        $fromDate = "";
        $toDate = "";
        $dates_range_str = "";
    }
    
    $fileName = $_SESSION['username'];
    if (!defined('CHART_MIN_SIZE')) define("CHART_MIN_SIZE", 1);
    
    //check if we need to search within savedsearch
    if (!empty($savedSearch) && strlen($savedSearch) > 0){
        $_REQUEST["search_function"] = true;
        require_once 'searchEng.php';
        $savedSearch = $mysqli->real_escape_string($savedSearch);
        $sql = "SELECT id FROM savedsearch WHERE name='$savedSearch' AND username='$username'";
        $res = sqlQuery($mysqli,$sql);
        $row = $res->fetch_object()->id;
        $savedSearchSqlId = " AND files.id in (SELECT docid FROM savedsearchid WHERE id=$row) ";
        $savedSearchId = "SELECT docid FROM savedsearchid WHERE id=$row";
        $ssid = $row;
    }else{
        $savedSearchId = "";
        $savedSearchSqlId = "";
    }
    
    $dateSQLFormat = "date";
    $dates_range = getAllDatesByRange($mysqli, $dateSQLFormat, $fromDate, $toDate, $isAdmin, $username, $savedSearchSqlId);
    $docs_and_dates = array();
    $occ_and_dates = array();
    $total_and_dates = array();
    $entity_pro_docs = array();
    $entity_pro_caption_docs = array();
    $entity_pro_caption_count = array();
    $entity_anti_docs = array();
    $entity_anti_caption_docs = array();
    $entity_anti_caption_count = array();
    
    $dates_range_size = sizeof($dates_range);
    $fromToDate = $dates_range[0] . " -- " . end($dates_range);
    
    if($dates_range_size<$columnNum){
        $columnNum = $dates_range_size;
    }
    
    
    if ($dates_range_size >= 1000){
        if (!defined('CHART_MAX_SIZE')) define("CHART_MAX_SIZE", 1000);     
    }
    else{
        if (!defined('CHART_MAX_SIZE')) define("CHART_MAX_SIZE", $dates_range_size);     
    }
    //How many elements in each array that will be created
    if($dates_range_size>0){
        $each_array_size = round(($dates_range_size / CHART_MAX_SIZE),0,PHP_ROUND_HALF_DOWN);
    }
    else{
        $each_array_size = 0;
    }
    
    //Cuts the array to the same amount of $each_array_size
    
    $new_dates_range = $dates_range;
    $all_ana_txt = 0;
    $all_ent_occ = 0;
    $all_ent_pro = 0;
    $all_ent_anti = 0;
    
    /**
     * Refactored Area
     */
    $max_caption = array_key_exists($entitiesSelection, $dropdownArrMaxMapping) || array_key_exists($entitiesSelection, $dropdownArrCaptionMapping);
    /*
     * fetch variables
     */
    // 	if(isEntity($entitiesSelection) || array_key_exists($entitiesSelection, $dropdownArr) || $entitiesSelection === 'concept'){
    global $extraDataKey2Table, $dropdownArrSolr, $dropdownArr, $group_trend_table;
    global $skipDuplicate;
    $useSolr = getSystemSettingsProp($mysqli, "useSolr");
    
    $_REQUEST['skip_include'] = true;
    require_once 'searchEng.php';
    if($max_caption){
        $caption_table = $dropdownArrMaxMapping[$entitiesSelection] ?? $dropdownArrCaptionMapping[$entitiesSelection];
        $orgSolrField_max = array_search($caption_table,$extraDataKey2Table);
    }
    $orgSolrField = array_search($entitiesSelection,$extraDataKey2Table);
    $response = array();
    $analytics_obj = new AnalyticsClass($mysqli, $caption_table ?? $entitiesSelection, $freeTextInput, $username, 0);
    $saved_search_arr = $analytics_obj->getSavedSearchName($savedSearch, true);
    $saved_search_name = $saved_search_arr["name"];
    $analytics_obj->setSavedSearchName($saved_search_name);
    
    //fetch the parameters for the search
    $params = $analytics_obj->fetch_params_search($saved_search_arr, $extraDataKey2Table, $useSolr, $dropdownArr, $dropdownArrSolr, $skipDuplicate, $orgSolrField_max ?? $orgSolrField, $savedSearch, false, false, true, null,null, false);
    /*
     * 1a - Entity selection - not person
     */
    if(isEntity($entitiesSelection) && $entitiesSelection !=="person_object" || $entitiesSelection == $group_trend_table){
        
        $id = $analytics_obj->search_and_fetch_id($params['all'] ?? $params, $saved_search_name, $dates_range_str);
        $results = sentimentTrend($id, $fileName, $entitiesSelection, $freeTextInput, $isAdmin, $username, $savedSearchSqlId, $dates_range, $dateSQLFormat, $mysqli);
        
    }
    /*
     * 1b - Entity selection - person
     */
    elseif ($entitiesSelection == "person_object"){
        
        $id = $analytics_obj->search_and_fetch_id($params['all'],$saved_search_name, $dates_range_str);
        $results = handle_Person_Trend($id, $mysqli, $freeTextInputStr, $entitiesSelection, $uniqueTable, $numberOfUsersTablesName, $dates_range, $dateSQLFormat, $username, $isAdmin, $savedSearchId, $savedSearchSqlId, $metadata_path, $fileName);
    }
    
    /*
     * 2 - $dropdownArr selection
     */
    elseif (array_key_exists($entitiesSelection, $dropdownArr)){
        $id = $analytics_obj->search_and_fetch_id($params,$saved_search_name, $dates_range_str);
        $results = handle_DropDown_Trend($id, $freeTextInput, $isAdmin, $username, $entitiesSelection, $dropdownArrMaxMapping, $dropdownArrCaptionMapping, $metadata_path, $fileName, $dates_range, $mysqli, $dateSQLFormat, $savedSearchSqlId);
    }
    
    /*
     * 3 - concept selection
     */
    elseif ($entitiesSelection === 'concept'){
        $id = $analytics_obj->search_and_fetch_id($params,$saved_search_name, $dates_range_str);
        $results = handle_Concept_Trend($id, $dates_range, $freeTextInput, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId, $metadata_path, $fileName, $all_ana_txt, $all_ent_occ, $total_and_dates, $docs_and_dates, $occ_and_dates);
    }
    /*
     * 4 - threat_rating selection - aint part of the semantic DB so no need refactor
     */
    
    elseif ($entitiesSelection === "threat_rating"){
        $response = getThreatRatingTrend(CHART_MAX_SIZE,$asAvg,$metadata_path,$fileName,$dates_range,$mysqli,$dateSQLFormat,$username, $isAdmin, $savedSearchSqlId,$each_array_size,$entitiesSelection);
    }
    //5 - if its Concepts general - make sure only general concept search go here V2
    elseif ($entitiesSelection === "saved_search"){
        $id = $ssid;
        $results = handle_savedSearch_Trend($mysqli, $username, $isAdmin, $savedSearch, $dates_range, $ssid, $savedSearchSqlId, $dateSQLFormat);
    }
    else{
        
        $id = $analytics_obj->search_and_fetch_id($params,$saved_search_name, $dates_range_str);
        $freeTextInput_tmp = $entitiesSelection . ';' . $freeTextInput;
        $results = handle_Concept_Trend($id, $dates_range, $freeTextInput_tmp, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId, $metadata_path, $fileName, $all_ana_txt, $all_ent_occ, $total_and_dates, $docs_and_dates, $occ_and_dates);
    }
    
    
    $response["lineGraphOptions"] = $lineGraphOptions;
    $response["barGraphOptions"] = $barGraphOptions;
    $response["uniqueTable"] = $uniqueTable;
    $response["xLabelPoints"] = $dates_range;
    
    $mood_array = mood_graph_handle_v2($dates_range, $mysqli, $isAdmin, $username, $id ?? null);
    
    // add mood counts to response array (v2 uses Factor and not pro-anti-total)
    $response["allMoodTotal"] = $mood_array['all'] ?? [];
    
    
    /*
     * fetched the varialbes from the results - from refactored area
     */
    $docs_and_dates = $results['docs_and_dates'] ?? [];
    $occ_and_dates = $results['occ_and_dates'] ?? [];
    $total_and_dates = $results['total_and_dates'] ?? [];
    $entity_pro_docs = $results['entity_pro_docs'] ?? [];
    $entity_pro_caption_docs = $results['entity_pro_caption_docs'] ?? [];
    $entity_pro_caption_count = $results['entity_pro_caption_count'] ?? [];
    $entity_anti_docs = $results['entity_anti_docs'] ?? [];
    $entity_anti_caption_docs = $results['entity_anti_caption_docs'] ?? [];
    $entity_anti_caption_count = $results['entity_anti_caption_count'] ?? [];
    $all_ent_occ = $results['all_ent_occ'] ?? [];
    $all_ent_pro = $results['all_ent_pro'] ?? [];
    $all_ent_anti = $results['all_ent_anti'] ?? [];
    $all_ana_txt = $results['all_ana_txt'] ?? [];
    //****
    
    if(empty($title)){
        $title = ucfirst($freeTextInput);
    }
    
    if ($entitiesSelection !== "threat_rating") {
        if(!empty($occ_and_dates)){
            $response["occ"] = $occ_and_dates;
        }
        
        $response["total"] = $total_and_dates ?? [];
        $response["anti_captions"] = $entity_anti_caption_docs ?? [];
        
        $response["pro_captions"] = $entity_pro_caption_docs ?? [];
        $response["anti_captions"] =  $entity_anti_caption_docs ?? [];
        $response["proPoints"] = $entity_pro_docs ?? [];
        $response["antiPoints"] = $entity_anti_docs ?? [];
        
        
        if ($asAvg == "true") {
            if (! empty($entity_pro_caption_count)) {
                $response["pro_occ"] = $entity_pro_caption_count;
            }
            if (! empty($entity_pro_caption_docs)) {
                $response["anti_occ"] = $entity_anti_caption_count;
            }
            $response["fromToDate"] = $fromToDate;
            $response["xLabelPoints"] = $new_dates_range;
            $response["asAvg"] = true;
        } else {
            $response["xLabelPoints"] = $dates_range;
            $response["asAvg"] = false;
        }
        $response["title"] = $title;
        $response["points"] = $docs_and_dates;
        $response["fileName"] = $fileName;
        $response["allEntTxt"] = $all_ent_occ;
        $response["allEntPro"] = $all_ent_pro;
        $response["allEntAnti"] = $all_ent_anti;
    }
    
    $response["allAnaTxt"] = $all_ana_txt;
    $response["isTotalMentionsHidden"] = $isTotalMentionsHidden;
    $response["isDateRangeLimitedToMentions"] = $isDateRangeLimitedToMentions;
    
    $response['savedSearchName'] = $saved_search_name ?? null;
    
    if(!empty($savedSearch)){
        if (strlen($savedSearch) > 0){
            $response["savedsearch"] = $savedSearch;
        }
    }
    
    if ($cache){
        CacheEng::updateResultsAndCache($mysqli, $username,  "trend", $title, $response);
    }
    else if ($print){
        echo json_encode($response);
    }
    return $response;
}

function getTrendGroup($mysqli, $username){
    global $group_trend_table;
    $elements_str = "";
    $data = file_get_contents('php://input');
    //     $data = "[{\"entityType\":\"organizational_identity\",\"entityId\":\"organizational_identity\",\"entityName\":\"Hamas\"},{\"entityType\":\"person_object\",\"entityId\":\"person_object\",\"entityName\":\"Sarr\"}]";
    
    $data_arr = json_decode($data,1);
    
    //start with reset previous cross entity of groupTrend
    $group_trend_sql = "SELECT id FROM concepts_indexes WHERE concept = 'group_trend' ";
    $group_trend_res = sqlQuery($mysqli, $group_trend_sql);
    if ($trend_row = $group_trend_res->fetch_object()){
        $group_trend_id = $trend_row->id;
    }
    $reset_sql = "DELETE FROM all_concepts_table WHERE concept_type=\"$group_trend_id\" AND username=\"$username\"";
    $reset_res = sqlQuery($mysqli, $reset_sql);
    
    //a - need to cross-entity the elements to seperate entity for trends
    require_once 'ManageEntitiesEng.php';
    foreach($data_arr as $element){
        $orig_table = $element['entityType'];
        $orig_name = $element['entityName'];
        $res = crossEntity($mysqli, $username, true, $orig_name,$orig_table, $group_trend_table, true);
        $elements_arr[] = $element['entityName'];
        
    }
    $elements_str = json_encode($elements_arr, true);
    //b - need to manageEntities those elements together
    require_once 'v2'.DIRECTORY_SEPARATOR .'ManageEntitiesEng_v2.php';
    $input = "{\"concept\":\"group_trend\",\"name\":\"group_trend_agg\",\"Entities\":" . $elements_str . "}";
    $data = json_decode($input,1) ;
    
    $res = insertData_v2($mysqli, $username, $data);
    
    //c - run trend on those entities
    require_once 'proSentPageEng.php';
    $_REQUEST["group_input"] = "{\"savedsearch\": \"Last Search\", \"TemplateName\": \"proSentResults\",\"asAvg\": true,\"collapse\": true,\"columnNum\": 20,\"entitiesSelection\": \"group_trend\",\"freeTextInput\": \"group_trend_agg\",\"fromDay\": 0,\"fromMonth\": 0,\"fromYear\": 0,\"intervalType\": \"all\",\"isDateRangeLimitedToMentions\": true,\"fromDate\": null, \"toDate\": null,\"topBannerDisplay\": \"none\"}";
    $response = getTrend($mysqli, true, null, null);
    
    //d - reset the entities back
    require_once 'ManageEntitiesEng.php';
    $res = deleteEntity_v2($mysqli, "group_trend_agg");
    
    
    
    return $response;
}

function handle_Concept_Trend($id, $dates_range, $freeTextInput, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId, $metadata_path, $fileName){
    
    $results = array('all_ana_txt'=>0, 'all_ent_occ'=>0, 'total_and_dates'=>array(), 'docs_and_dates'=>array(), 'occ_and_dates'=>array());
    $savedSearch_sql = "Select docId FROM savedsearchid WHERE id=$id";
    
    $params = explode(";",$freeTextInput);
    if ($params[0] === "conceptType"){
        $table = strtolower($params[1]);
        $conceptName = $params[2];
        $type = true;
    }else{
        $table = strtolower($params[0]);
        $conceptName = $params[1];
        $type = false;
    }
    
    $isAll = false;
    if($conceptName==="All" || empty($conceptName)){
        $isAll = true;
    }
    
    file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv", "Date;Documents\n");
    $appendToFile = "";
    //get number of all analyzed docs per each date at range - total number
    $date2allDocs = getNumofAllAnalayzedDocsPerDatesRange($dates_range, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId);
    
    //get the dates, docs and occs for the entity selection
    $entity_arr = getDocIdsFromConcept($savedSearch_sql, $table, $conceptName ,$dates_range, $mysqli, $dateSQLFormat,  $type, $isAdmin, $username, $isAll, $savedSearchSqlId);
    
    //$entitiesSelection is a name of a column
    foreach ($dates_range as $date){
        
        //Sum of docs that were analayzed in this date
        if (!array_key_exists($date, $date2allDocs)){
            write_to_log("error", "missing date: $date in " . print_r($date2allDocs, true));
        }
        
        $all_date_docs = $date2allDocs[$date];
        $results['all_ana_txt'] += $all_date_docs;
        //Count how many docs were analayzed with this entity appeared in this date
        if(empty($savedSearchSqlId)){
            $savedSearchSqlId = "";
        }
        //Returns an array of documents ids of an entity in a single date.
        if (!array_key_exists($date, $entity_arr)){
            write_to_log("TRACE", "date doesn't contain the element - skipping");
        }
        
        $entity_arr_date = $entity_arr[$date] ?? null;
        
        $entity_occ = $entity_arr_date["occ"] ?? 0;
        $entity_docs = $entity_arr_date["doc"] ?? 0;
        $results['all_ent_occ'] += $entity_docs;
        
        //Black line
        //insert the points
        $results['total_and_dates'][] = array($date,$all_date_docs);
        $results['docs_and_dates'][] = array($date,$entity_docs);
        $results['occ_and_dates'][] = array($date,$entity_occ);
        
        $appendToFile .= "$date;$entity_docs\n";
    }
    
    file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv" , $appendToFile,FILE_APPEND);
    
    return $results;
}

function calculateGprahPointsSavedSearch($mysqli, $savedSearch_sql, $params){
    $params = json_decode($params, true);
    $dots_sql = "SELECT COUNT(files.id) as count, date(files.date) as date FROM files WHERE files.id IN ($savedSearch_sql)  GROUP BY YEAR(files.date), MONTH(files.date), DAY(files.date)";
    $res = sqlQuery($mysqli,$dots_sql);
    while (!empty($res) && ($row = $res->fetch_object())){
        $dots_data['amount'][$row->date] =intval($row->count);
    }
    $dots_data['positive'] = getSentimentPerDateAccum($mysqli, 'positive', $savedSearch_sql, $params);
    $dots_data['negative'] = getSentimentPerDateAccum($mysqli, 'anti', $savedSearch_sql, $params);
    return $dots_data;
}

function getSentimentPerDateAccum($mysqli, $sentiment, $savedSearch_sql, $params){
    $uvTable = "";
    $fieldName = "f.id";
    $uvCondition = "";
    $date2sent_docs = [];
    $sql_sent_captions = ", GROUP_CONCAT(DISTINCT t.$sentiment  SEPARATOR ', ') AS ".strtoupper($sentiment)."_caption";
    $sent_table = $sentiment == 'positive' ? "pro" : "anti";
    foreach($params as $concept=>$instances){
        if(!TablesInformation::isTableExists($mysqli, $concept) || !TablesInformation::isColumnExists($mysqli, $concept, 'name')){
            continue;
        }
        foreach($instances as &$instance){
            $instance = $mysqli->real_escape_string($instance);
        }
        $sql_sent =  "SELECT COUNT(DISTINCT(f.id)) AS $sentiment, CAST(f.date AS DATE) AS date_col, SUM(t.occurrences)
        AS occ , GROUP_CONCAT(DISTINCT t.$sentiment  SEPARATOR ', ') AS ".strtoupper($sentiment)."_caption
        FROM files AS f, $concept AS t,".$sent_table."_instance AS pms
        WHERE f.id IN ($savedSearch_sql)
        AND t.docId = f.id AND t.name IN ('".implode("','", $instances) . "')
                     AND t.id = pms.instanceId GROUP BY date_col ";
        
        $res = sqlQuery($mysqli, $sql_sent);
        
        while($row = $res->fetch_object()){
            $sent_captions = strtoupper($sentiment).'_caption';
            //if the date already in the answer array
            if(array_key_exists($row->date_col, $date2sent_docs)){
                $date2sent_docs[$row->date_col]['sum'] += intval($row->$sentiment);
                $date2sent_docs[$row->date_col]['captions'][] = $row->$sent_captions ?? "";
                $date2sent_docs[$row->date_col][$sentiment.'_count']+= intval($row->occ);
            }
            //if the date is not in the answer array
            else{
                $date2sent_docs[$row->date_col]['sum'] = intval($row->$sentiment);
                $date2sent_docs[$row->date_col]['captions'][] = arrangeCaptions($row->$sent_captions ?? []);
                $date2sent_docs[$row->date_col][$sentiment.'_count'] = intval($row->occ);
            }
        }
        
    }
    
    
    return $date2sent_docs;
}

function getNegativeDateAccum($mysqli, $params){
    foreach($params as $concepts=>$instances){
        
    }
}


function calculateGprahPoints($mysqli, $savedSearch_sql, $table,$name,$isAdmin,$username, $mappedCaption = null){
    $table = strtolower($table);
    $user_sql = !$isAdmin ? "  files.username='$username'" : "TRUE";
    if(!empty($mappedCaption)){
        $dots_sql="SELECT COUNT($table.docId) as count, date(files.date) as date FROM $table,files, 
        $mappedCaption WHERE files.id IN ($savedSearch_sql) AND $mappedCaption.value='$name' AND
        $mappedCaption.docId = $table.docId AND files.id=$table.docId AND $user_sql GROUP BY 
        YEAR(files.date), MONTH(files.date), DAY(files.date)";
    }
    else{
//         $dots_sql = "SELECT COUNT($table.docId) as count, date(files.date) as date FROM $table,files
//              WHERE files.id IN 
//             ($savedSearch_sql) AND $table.value='$name' AND files.id=$table.docId AND 
//         $user_sql GROUP BY YEAR(files.date), MONTH(files.date), DAY(files.date)";
        $dots_sql = "SELECT COUNT($table.docId) as count,
        DATE(files.date) as date
        FROM $table
        INNER JOIN files ON files.id = $table.docId
        WHERE files.id IN ($savedSearch_sql)
        AND $table.value = '$name'
        AND $user_sql
        GROUP BY DATE(files.date)";
    }
    $res = sqlQuery($mysqli,$dots_sql);
    while (!empty($res) && ($row = $res->fetch_object())){
        $dots_data[$row->date] =intval($row->count);
    }
    return $dots_data;
}


function handle_savedSearch_Trend($mysqli, $username, $isAdmin, $savedSearchName, $dates_range, $ssid, $savedSearchSqlId, $dateSQLFormat){
    $results = array('all_ana_txt'=>0, 'all_ent_occ'=>0, 'total_and_dates'=>array(), 'docs_and_dates'=>array(), 'occ_and_dates'=>array());
    $params_sql = "SELECT search_values FROM savedsearch where id=$ssid";
    $res = sqlQuery($mysqli, $params_sql);
    $obj = $res->fetch_object();
    $params = $obj->search_values;
    $savedSearch_sql = "Select docId FROM savedsearchid WHERE id=$ssid";
    
    //gets all the points per data in the graph
    $calc_arr = calculateGprahPointsSavedSearch($mysqli, $savedSearch_sql, $params);
    $dates_full_data = $calc_arr["amount"];
    $dates_pos_data = $calc_arr["positive"];
    $dates_neg_data = $calc_arr["negative"];
    
    $date2allDocs = getNumofAllAnalayzedDocsPerDatesRange($dates_range, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId);
    
    //$entitiesSelection is a name of a column
    foreach ($dates_range as $date){
        //Sum of docs that were analayzed in this date
        if (!array_key_exists($date, $date2allDocs)){
            write_to_log("error", "missing date: $date in " . print_r($date2allDocs, true));
        }
        
        $all_date_docs = $date2allDocs[$date];
        $results['all_ana_txt'] += $all_date_docs;
        
        //skip if not setup
        if(empty($dates_full_data[$date])){
            $dates_full_data[$date] = 0;
        }
        $entity_docs_sum = $dates_full_data[$date];
        
        $tmp_entity_docs_sum = 0;
        
        $entity_docs_sum += $tmp_entity_docs_sum;
        
        $results['all_ent_occ'] += $entity_docs_sum;
        //Black line
        array_push($results['total_and_dates'], array($date,$all_date_docs));
        array_push($results['docs_and_dates'], array($date,$entity_docs_sum));
        
        //Green and Light green lines
        $results['entity_pro_docs'][] = array($date,$dates_pos_data[$date]['sum'] ?? 0);
        $pro_captions = $dates_pos_data[$date]["captions"] ?? "";
        $results['entity_pro_caption_docs'][] = array($date,$pro_captions);
        $results['entity_pro_caption_count'][] = array($date,$dates_pos_data[$date]['captions_count'] ?? 0);
        
        //Red and light red lines
        $results['entity_anti_docs'][] = array($date,$dates_neg_data[$date]['sum']  ?? 0);
        $anti_captions = $dates_neg_data[$date]["captions"] ?? "";
        $results['entity_anti_caption_docs'][] = array($date,$anti_captions);
        $results['entity_anti_caption_count'][] = array($date,$dates_neg_data[$date]['captions_count'] ?? 0);
        
    }
    return $results;
}

function handle_DropDown_Trend($id, $freeTextInput, $isAdmin, $username, $entitiesSelection, $dropdownArrMaxMapping, $dropdownArrCaptionMapping, $metadata_path, $fileName, $dates_range, $mysqli, $dateSQLFormat, $savedSearchSqlId){
    $results = array('all_ana_txt'=>0, 'all_ent_occ'=>0, 'total_and_dates'=>array(), 'docs_and_dates'=>array(), 'occ_and_dates'=>array());
    
    $savedSearch_sql = "Select docId FROM savedsearchid WHERE id=$id";
    
    //checked if we have a mapped caption - different handling
    $mapped_caption = $dropdownArrCaptionMapping[$entitiesSelection] ?? $dropdownArrMaxMapping[$entitiesSelection] ?? null;
    
    //gets all the points per data in the graph
    $dates_full_data= calculateGprahPoints($mysqli, $savedSearch_sql, $entitiesSelection,$freeTextInput,$isAdmin,$username, $mapped_caption);
    $tempfreeTextInput = $freeTextInput;
    if ($entitiesSelection === 'counted_theatre'){
        $tempfreeTextInput = analyzeWord($freeTextInput, 'Place-object');
    }
    
    file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv", "Date;Documents\n");
    $appendToFile = "";
    
    $date2allDocs = getNumofAllAnalayzedDocsPerDatesRange($dates_range, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId);
    
    //$entitiesSelection is a name of a column
    foreach ($dates_range as $date){
        //Sum of docs that were analayzed in this date
        if (!array_key_exists($date, $date2allDocs)){
            write_to_log("error", "missing date: $date in " . print_r($date2allDocs, true));
        }
        
        $all_date_docs = $date2allDocs[$date];
        $results['all_ana_txt'] += $all_date_docs;
        
        //skip if not setup
        if(empty($dates_full_data[$date])){
            $dates_full_data[$date] = 0;
        }
        $entity_docs_sum=$dates_full_data[$date];
        
        $tmp_entity_docs_sum = 0;
        if($tempfreeTextInput !== $freeTextInput){
            $tmp_entity_docs_sum = $dates_full_data[$date];
        }
        $entity_docs_sum += $tmp_entity_docs_sum;
        
        $results['all_ent_occ'] += $entity_docs_sum;
        //Black line
        array_push($results['total_and_dates'], array($date,$all_date_docs));
        array_push($results['docs_and_dates'], array($date,$entity_docs_sum));
        
        $appendToFile .= "$date;$entity_docs_sum\n";
    }
    
    file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv" , $appendToFile,FILE_APPEND);
    
    return $results;
}

function handle_Person_Trend($id, $mysqli, $freeTextInputStr, $entitiesSelection, $uniqueTable, $numberOfUsersTablesName, $dates_range, $dateSQLFormat, $username, $isAdmin, $savedSearchId, $savedSearchSqlId, $metadata_path, $fileName){
    $results = array('all_ana_txt'=>0, 'all_ent_occ'=>0, 'total_and_dates'=>array(), 'docs_and_dates'=>array(), 'occ_and_dates'=>array(),
        'entity_pro_docs' => array(), 'entity_pro_caption_docs' => array(), 'entity_pro_caption_count' => array(), 'all_ent_pro' => 0,
        'entity_anti_docs' => array(), 'entity_anti_caption_docs' => array(), 'entity_anti_caption_count' => array(), 'all_ent_anti' => 0);
    
    $perMegaSqls = buildSearchEntitiesSql($mysqli, array(array("text" => $freeTextInputStr, "type" => "person,email")));
    $searchPerSql = buildFinalSql($perMegaSqls, $mysqli, NULL);
    $docIds_str= "SELECT docId FROM ($searchPerSql )as docIds WHERE docId IN(Select docId FROM savedsearchid WHERE id=$id)";
    
    $uniqueValueTableName = $uniqueTable == "username" ? $numberOfUsersTablesName : NULL;
    $date2allDocs = getNumofAllAnalayzedDocsPerDatesRange($dates_range, $mysqli, $dateSQLFormat, $username, $isAdmin, $savedSearchSqlId, $uniqueValueTableName);
    $date2entity_pro_sent = personSentimentPerDate("pro", $docIds_str, $dates_range, $searchPerSql, $entitiesSelection, $mysqli, $dateSQLFormat, $savedSearchId, $username, $isAdmin, $uniqueValueTableName);
    $date2entity_anti_sent= personSentimentPerDate("anti", $docIds_str, $dates_range, $searchPerSql, $entitiesSelection, $mysqli, $dateSQLFormat, $savedSearchId, $username, $isAdmin, $uniqueValueTableName);
    $date2per_docs = personSentimentPerDate("total", $docIds_str, $dates_range, $searchPerSql, $entitiesSelection, $mysqli, $dateSQLFormat, $savedSearchId, $username, $isAdmin, $uniqueValueTableName);
    
    $appendToFile = "";
    foreach ($dates_range as $date_r){
        if (!array_key_exists($date_r, $date2allDocs)){
            write_to_log("error", "missing date: $date_r in " . print_r($date2allDocs, true));
        }
        $all_date_docs = $date2allDocs[$date_r];
        $results['all_ana_txt'] += $all_date_docs;
        
        $per_sum_date_docs_arr = key2val($date2per_docs, $date_r);
        if(empty($per_sum_date_docs_arr)){
            $per_sum_date_docs_arr[$date_r] = 0;
        }
        $per_occ_date_docs = $per_sum_date_docs_arr['occ'] ?? 0;
        $per_sum_date_docs = $per_sum_date_docs_arr['total'] ?? 0;
        
        $results['all_ent_occ'] += $per_sum_date_docs;
        
        //Black line $inner_docs_and_dates
        $results['total_and_dates'][] =  array($date_r,$all_date_docs);
        $results['docs_and_dates'][] = array($date_r,$per_sum_date_docs);
        $results['occ_and_dates'][] = array($date_r,$per_occ_date_docs);
        
        //Green and Light green lines
        $pro_date = key2val($date2entity_pro_sent, $date_r);
        
        $entity_pro_sent = empty($pro_date) ? "" : personSentiment("pro", $pro_date);
        $results['entity_pro_docs'][] = array($date_r,$entity_pro_sent["general"] ?? 0);
        
        $pro_captions = $entity_pro_sent["captions"] ?? "";
        
        $results['entity_pro_caption_docs'][] = array($date_r,$pro_captions);
        
        $results['entity_pro_caption_count'][] = array($date_r,$entity_pro_sent['captions_count'] ?? 0);
        
        if(!empty($entity_pro_sent)){
            $results['all_ent_pro'] += $entity_pro_sent["totalPro"];
        }
        
        //Red and light red lines
        $anti_date = key2val($date2entity_anti_sent, $date_r);
        $entity_anti_sent = empty($anti_date) ? "" : personSentiment("anti", $anti_date);
        $results['entity_anti_docs'][] = array($date_r,$entity_anti_sent["general"] ?? 0);
        
        $anti_captions = $entity_anti_sent["captions"] ?? "";
        
        $results['entity_anti_caption_docs'][] = array($date_r,$anti_captions);
        $results['entity_anti_caption_count'][] = array($date_r,$entity_anti_sent['captions_count'] ?? 0);
        
        if(!empty($entity_anti_sent)){
            $results['all_ent_anti'] += $entity_anti_sent["totalAnti"];
        }
        
        //Writing to file
        $pro_to_file = empty($entity_pro_sent) ? 0 : $entity_pro_sent["general"];//$entity_pro_sent["general"][1];
        $anti_to_file = empty($entity_anti_sent) ? 0 : $entity_anti_sent["general"];//$entity_anti_sent["general"][1];
        
        $appendToFile .= "$date_r;$per_sum_date_docs;" . $pro_to_file . ";" . $anti_to_file . "\n";
        
        unset($entity_pro_sent);
        unset($entity_anti_sent);
        
    }
    file_put_contents("$metadata_path".DIRECTORY_SEPARATOR . $fileName . ".csv" , $appendToFile,FILE_APPEND);
    return $results;
}
