<?php

require_once 'globalArr.php';
require_once 'slotToCaption.php';
require_once 'concept.php';
require_once 'demosettings.php';

require_once 'allConceptsData.php';

/**
 * init_mainEng - get username, permissions, action and sql Connection
 */
function init_mainEng(){
    require_once 'allConceptArr.php';
    require_once 'sqlUtils.php';   
    require_once 'parseUtils.php';
    require_once 'settings.php';

    
    $userdetails = getUserDetails();
    $username = $userdetails['Username'];
    $isAdmin = getUserPermission(CAN_SHOW_ALL_USER) ? 1 : 0;
    $action = $_REQUEST["Action"];
    $mysqli = sqlCreateConnection("init_mainEng - $action");
    
    $settings_class =  new Settings($mysqli, $username);
    $allConceptArrDB = $settings_class->getAllConceptsArr();
    
    $concept_class = new AllConceptsData($mysqli, $allConceptArrDB, null, $username);
    
    return [
        'username' => $username,
        'isAdmin' => $isAdmin,
        'action' => $action,
        'mysqli' => $mysqli,
        'concept_class' => $concept_class
    ] ;
}

function getSlotByTable($connection, $request){
    require_once 'sqlUtils.php';
    return (getSlotD($connection, $request["name"], $request["table"], null));
}

function getDateLastModified($mysqli){
    require_once 'sqlUtils.php';
    require_once 'parseUtils.php';
    $username = $_SESSION['username'];
    
    $status = getRunningStatus($mysqli,$username);
    if($status>0){
        $date = date('YmdHis');
    }else{
        $query = "SELECT last_time_modified AS l FROM user_status WHERE username='$username'";
        $results = sqlQuery($mysqli,$query);
        if(!empty($results)){
            $row = $results->fetch_object();
            $date = $row->l;
        }
    }
    return $date;
}


function listFilesPerDir($connection, $id, $username, $isAdmin, $numberToFetch = null, $offset = null){
    $limitResult = 500;
    
    
    $res = sqlQuery($connection, "SELECT name,parentId FROM foldertree WHERE id=$id");
    $obj = $res->fetch_object();
    if (strpos($obj->name,'save_search::') !== false)
        $response["saveSearch"] = true;
        if (strpos($obj->name,'save_search::Static') !== false){
            //////////////////
            ////Static search
            /////////////////
            $res = sqlQuery($connection, "SELECT name FROM foldertree where id = " . $obj->parentId);
            if ($res)
            {
                $name = $res->fetch_object()->name;
                $e = explode('save_search::',$name);
                $name = $e[1];
                $res = sqlQuery($connection, "SELECT id FROM savedsearch where name='$name' AND username='$username'");
                $id = $res->fetch_object()->id;
                
                $sql = "SELECT docid FROM savedsearchid WHERE id=$id LIMIT $limitResult";
                $res = sqlQuery($connection, $sql);
                
                $lastSearch = "";
                $first = true;
                while ($obj = $res->fetch_object()){
                    if ($first){
                        $first = false;
                        $lastSearch = $obj->docid;
                    }
                    else{
                        $lastSearch .= "," . $obj->docid;
                    }
                }
                
                $sql = "SELECT * FROM files WHERE id IN ($lastSearch) ";
                
                if(isset($numberToFetch) && isset($offset)){
                    $sql .= " LIMIT $numberToFetch OFFSET $offset ";
                }
                else{
                    $sql .= "LIMIT $limitResult";
                }
                
                $res = sqlQuery($connection, $sql);
            }
        }else if (strpos($obj->name,'save_search::Dynamic') !== false){
            //////////////////
            ////Dynamic search
            /////////////////
            $res = sqlQuery($connection, "SELECT name FROM foldertree where id = " . $obj->parentId);
            $_REQUEST["search_function"] = true;
            require_once 'searchEng.php';
            $name = $res->fetch_object()->name;
            $e = explode('save_search::',$name);
            $name = $connection->real_escape_string($e[1]);
            
            $arr_s = search($mysqli, $name);
            $lastSearch = implode(",", $arr_s);
            $sql = "SELECT * FROM files WHERE id IN ($lastSearch) LIMIT $limitResult";
            $res = sqlQuery($connection, $sql);
        }
        else{
            
            $arr_id = array();
            
            $sql = "SELECT * FROM files where parentFolderId = $id ORDER BY filename";
            
            if(isset($numberToFetch) && isset($offset)){
                $sql .= " LIMIT $numberToFetch OFFSET $offset ";
            }
            else{
                $sql .= " LIMIT $limitResult";
            }
            
            $res = sqlQuery($connection, $sql);
            
            if ($res){
                while ($obj = $res->fetch_object())
                {
                    $obj->docId = $obj->id;
                    
                    $obj->name = $obj->filename;
                    $arr_id[$obj->id] = $obj;
                }
            }
            
        }
        $arr_id = array_values($arr_id);
        global $dropdownArr;
        if (!empty($arr_id)){
            $arr_meta = getMetaDataSpecielArrFormDB($connection, $arr_id, $limitResult,$username, $isAdmin, null, null, $numberToFetch, $offset);
            
            $arr_review = getReviewed($connection, $arr_id, $limitResult, null,null,  $numberToFetch, $offset);
            foreach ($arr_id as $k => &$v){
                foreach ($dropdownArr as $meta_k => $meta_v){
                    if (array_key_exists($meta_k, $arr_meta) && array_key_exists($v->id, $arr_meta[$meta_k]) && $meta_k !== "emotion"){
                        $v->{$meta_k} = $arr_meta[$meta_k][$v->id];
                    }
                }
                $v->reviewed = array_key_exists($v->id, $arr_review)? true: false;
                $v->redacted = ($v->redactedId == NULL)? false : true;
                $v = moodHandleData($v, $v->id, $arr_meta);
                
            }
        }
        
        if (empty($arr_id))
        {
            $arr_id[] = ["id" => 1, "filename" => "files not found"];
        }
        
        return $arr_id;
}

function   moodHandleData($v, $docid, $metaArr){
    $fields = array("total_emotion", "anti_emotion", "pro_emotion", "mood");
    foreach($fields as $f){
        $v->{$f} = $metaArr[$f][$docid] ?? null;
    }
    return $v;
}

function listFoldersTree($connection, $parentFolderId)
{
    require_once 'sqlUtils.php';
    $limitResult = 500;
    //global $limitResult,
    global $dropdownArr;
    if ($parentFolderId === NULL)
        return array("error" => "no folder specified");
        
        $username = $_SESSION['username'];
        $emailFolders = array();
        $response = array();
        $isAdmin = getUserPermission(CAN_SHOW_ALL_USER);
        $arr_id = array();
        $arr_files = array();
        
        $query = "SELECT id, Name FROM foldertree where parentId = $parentFolderId";
        
        if($parentFolderId == 0 && !$isAdmin){
            $query .=" AND username='$username'";
        }else{
            $query .=" ORDER BY Name";
        }
        
        $query .= " LIMIT $limitResult";
        
        $res = sqlQuery($connection,$query);
        if ($res){
            while ($obj = $res->fetch_object())
            {
                //checks if dir has children
                $has_children = false;
                $query = "SELECT id, parentId, Name FROM foldertree WHERE parentId = $obj->id";
                $children_res = sqlQuery($connection,$query);
                if($children_obj = $children_res->fetch_object()){
                    $has_children = true;
                }
                
                $e = explode('save_search::',$obj->Name);
                if (sizeof($e) == 2){
                    $name = $e[1];
                }
                else{
                    $name = $e[0];
                }
                $object = (object) [
                    'id' => $obj->id,
                    'name' => $name,
                    'hasChildren' => $has_children
                ];
                
                $array = json_decode(json_encode($object), true);
                
                $response["folders"][] = $array;
               
            }
        }
        
     
    return ($response['folders']);
}

function listDirectory($connection, $parentFolderId, $numberToFetch = null, $offset = null)
{
    require_once 'sqlUtils.php';  
    $limitResult = 500;
    //global $limitResult,
    global $dropdownArr, $whatsapps_ext;
    if ($parentFolderId === NULL){
        return array("error" => "no folder specified");
    }
        
    $username = $_SESSION['username'];
    $emailFolders = array();
    $response = array();
    $isAdmin = getUserPermission(CAN_SHOW_ALL_USER);
    $arr_id = array();
    $arr_files = array();
    
    $query = "SELECT id, Name FROM foldertree where parentId = $parentFolderId";
    
    if($parentFolderId == 0 && !$isAdmin){
        $query .=" AND username='$username'";
    }else{
        $query .=" ORDER BY Name";
    }
    
    $query .= " LIMIT $limitResult";
    
    $res = sqlQuery($connection,$query);
    if ($res){
        while ($obj = $res->fetch_object())
        {
            $e = explode('save_search::',$obj->Name);
            if (sizeof($e) == 2){
                $name = $e[1];
            }
            else{
                $name = $e[0];
            }
            $response["folders"][] = array("id" => $obj->id, "name" => $name);
            if (isEmailfolder($obj->Name))
            {
                $emailFolders[] = $obj->id;
            }
        }
    }
    
    $res = sqlQuery($connection, "SELECT name,parentId FROM foldertree where id = $parentFolderId");
    $obj = $res->fetch_object();
    if(!empty($obj) && array_key_exists("name", $obj)){
    if (strpos($obj->name,'save_search::') !== false)
        $response["saveSearch"] = true;
        if (strpos($obj->name,'save_search::Static') !== false){
            //////////////////
            ////Static search
            /////////////////
            $res = sqlQuery($connection, "SELECT name FROM foldertree where id = " . $obj->parentId);
            if ($res)
            {
                $name = $res->fetch_object()->name;
                $e = explode('save_search::',$name);
                $name = $e[1];
                $res = sqlQuery($connection, "SELECT id FROM savedsearch where name='$name' AND username='$username'");
                $id = $res->fetch_object()->id;
                
                $sql = "SELECT docid FROM savedsearchid WHERE id=$id LIMIT $limitResult";
                $res = sqlQuery($connection, $sql);
                
                $lastSearch = "";
                $first = true;
                while ($obj = $res->fetch_object()){
                    if ($first){
                        $first = false;
                        $lastSearch = $obj->docid;
                    }
                    else{
                        $lastSearch .= "," . $obj->docid;
                    }
                }
                
                $sql = "SELECT * FROM files WHERE id IN ($lastSearch) ";
                
                if(isset($numberToFetch) && isset($offset)){
                    $sql .= " LIMIT $numberToFetch OFFSET $offset ";
                }
                else{
                    $sql .= "LIMIT $limitResult";
                }
                
                $res = sqlQuery($connection, $sql);
            }
        }else if (strpos($obj->name,'save_search::Dynamic') !== false){
            //////////////////
            ////Dynamic search
            /////////////////
            $res = sqlQuery($connection, "SELECT name FROM foldertree where id = " . $obj->parentId);
            $_REQUEST["search_function"] = true;
            require_once 'searchEng.php';
            $name = $res->fetch_object()->name;
            $e = explode('save_search::',$name);
            $name = $connection->real_escape_string($e[1]);
            
            $arr_s = search($connection, $name);
            $lastSearch = implode(",", $arr_s);
            $sql = "SELECT * FROM files WHERE id IN ($lastSearch) LIMIT $limitResult";
            $res = sqlQuery($connection, $sql);
        }
    
        else{
            $sql = "SELECT * FROM files where parentFolderId = $parentFolderId ORDER BY filename";
            
            if(isset($numberToFetch) && isset($offset)){
                $sql .= " LIMIT $numberToFetch OFFSET $offset ";
            }
            else{
                $sql .= " LIMIT $limitResult";
            }
            
            $res = sqlQuery($connection, $sql);
        }
    }
        
        $arr_id = array();
        $whatsapp_fromat = true;
        if ($res){
            while ($obj = $res->fetch_object())
            {
                //if theres one file not in split/original format - dont do the whatsapp oder of files
                $ext = pathinfo($obj->filename, PATHINFO_EXTENSION);
                
                if(!in_array($ext, $whatsapps_ext)){
                    $whatsapp_fromat = false;
                }
                $arr_id[$obj->id] = $obj->id;
                $obj->name = $obj->filename;
                $arr_files[$obj->id] = $obj;
            }
        }
        //if $whatsapp_format - order it by the order
        if($whatsapp_fromat){
            foreach($arr_files as &$file){
                $file_name = $file->filename;
                $id = (int) filter_var($file_name, FILTER_SANITIZE_NUMBER_INT);
                $file->order_id = $id; //(int) filter_var($file, FILTER_SANITIZE_NUMBER_INT);
            }
            usort($arr_files, function ($item1, $item2) {
                return $item1->order_id <=> $item2->order_id;
            });
        }
        
        //	if (!empty($emailFolders))
            // 	{
            // 		$sql = "SELECT * FROM files WHERE parentFolderId IN (" . join(", ", $emailFolders) . ") AND filename LIKE 'email%.%'";
        
            // 		if(isset($numberToFetch) && isset($offset)){
            // 		    $sql .= " LIMIT $numberToFetch OFFSET $offset ";
            // 		}
            // 		else{
            // 		    $sql .= " LIMIT $limitResult";
            // 		}
        
            // 		$res = sqlQuery($connection, $sql);
            // 		if ($res){
            //     		while ($obj = $res->fetch_object())
                //     		{
                //     			$arr_id[$obj->id] = $obj->id;
                //     			$obj->name = $obj->filename;
                //     			$arr_files[$obj->id] = $obj;
                //     		}
            // 		}
            // 	}
        
        if (!empty($arr_id)){
            $arr_meta = getMetaDataSpecielArrFormDB($connection, $arr_id, $limitResult,$username, $isAdmin, null, null, $numberToFetch, $offset);
            
            $arr_review = getReviewed($connection, $arr_id, $limitResult, null,null,  $numberToFetch, $offset);
            foreach ($arr_files as $k => $v){
                foreach ($dropdownArr as $meta_k => $meta_v){
                    if (array_key_exists($meta_k, $arr_meta) && array_key_exists($v->id, $arr_meta[$meta_k])){
                        $v->{$meta_k} = $arr_meta[$meta_k][$v->id];
                    }
                }
                $v->reviewed = array_key_exists($v->id, $arr_review)? "Yes" : "No";
                $v->redacted = ($v->redactedId == NULL)? "No" : "Yes";
                $response["files"][] = $v;
            }
        }
        
        if (empty($response["files"]))
        {
            $response["files"][] = ["id" => 1, "filename" => "files not found"];
        }
        
        return $response;
}

function fetchMoreDocuments($mysqli){
    $folder_id = $_REQUEST['folderId'] ?? null;
    $numberToFetch = $_REQUEST['numberToFetch'];
    $offset = $_REQUEST['offset'];
    return listDirectory($mysqli, $folder_id, $numberToFetch, $offset);
}

function deleteDirectory($connection, $folderId){
    require_once 'sqlUtils.php';
    $username = $_SESSION['username'];
    $folderName = "";
    
    $query = "SELECT name,username,parentid FROM foldertree where id = $folderId"; 
    $query .=" AND username='$username'";
    
    $res = sqlQuery($connection,$query);
    if ($res){
        $obj = $res->fetch_object();
        $folderName = $obj->username;
        $parentid = $obj->parentid;
        if (strpos($obj->name,'save_search::Static') !== false || strpos($obj->name,'save_search::Dynamic') !== false){
            $query = "SELECT name FROM foldertree where id = $parentid";
            $res = sqlQuery($connection,$query);
            $obj = $res->fetch_object();
            $e = explode('save_search::',$obj->name);
            $name = $e[1];
            //delete from foldertree
            $query = "DELETE FROM foldertree where id = $parentid OR parentid=$parentid";
            $res = sqlQuery($connection,$query);
            
            //delete from svaedsearch
            $query = "SELECT id FROM savedsearch where name = '$name' AND username='$folderName'";
            $res = sqlQuery($connection,$query);
            $id = $res->fetch_object()->id;
            $query = "DELETE FROM savedsearch where name = '$name' AND username='$folderName'";
            $res = sqlQuery($connection,$query);
            $query = "DELETE FROM savedsearchid where id = $id";
            $res = sqlQuery($connection,$query);
            
            $query = "DELETE FROM foldertree where id = $folderId OR parentd=$folderId";
        }elseif (strpos($obj->name,'save_search::') !== false){
            //delete from foldertree
            $query = "DELETE FROM foldertree where id = $folderId OR parentid=$folderId";
            $res = sqlQuery($connection,$query);
            //delete from svaedsearch
            $e = explode('save_search::',$obj->name);
            $name = $e[1];
            $query = "SELECT id FROM savedsearch where name = '$name' AND username='$folderName'";
            $res = sqlQuery($connection,$query);
            $id = $res->fetch_object()->id;
            $query = "DELETE FROM savedsearch where name = '$name' AND username='$folderName'";
            $res = sqlQuery($connection,$query);
            $query = "DELETE FROM savedsearchid where id = $id";
            $res = sqlQuery($connection,$query);
        }
    }
    if (strlen($folderName) == 0)
        $folderName = $username;
        return array("folderPath" => "/$folderName/savedsearch");
}
function isLocked($mysqli, $docId){
    write_to_log("PERMORMENCE", "Start isLocked - $docId" . date('H:i:s'.substr((string)microtime(), 1, 8)));
    global $userdetails,$AUTH;
    $AUTH = getGroupsAuth();
    
    if (array_key_exists("canContribute", $AUTH[$userdetails['Group_ID']]['permissions'])){
        if ($AUTH[$userdetails['Group_ID']]['permissions']["canContribute"] == false)
            return array("isLocked" => false,
                "user" => "");
    }else{
        return array("isLocked" => false,
            "user" => "");
    }
    require_once 'parseUtils.php';
    $res = isLockedDoc($mysqli, $docId,$_SESSION['username']);
    write_to_log("PERMORMENCE", "End isLocked - $docId" . date('H:i:s'.substr((string)microtime(), 1, 8)));
    return array("isLocked" => $res["status"],
        "user" => $res["user"]);
}
function releaseLocked($mysqli){
    require_once 'parseUtils.php';
    releaseLockDoc($mysqli, $_SESSION['username']);
    return array("status" => true);
}
//If folder is empty we know that request came from search
//If folder is not empty the request came from files page
function nextDoc($connection, $docId,$folder){
    write_to_log("PERMORMENCE", "Start nextDoc - $docId" . date('H:i:s'.substr((string)microtime(), 1, 8)));
    $username = $_SESSION['username'];
    if (!empty($folder) && strtolower($folder) !== 'null'){

        //////////////////////////////////
        //
        //	A - get next doc from folder
        //
        //////////////////////////////////
        $e_f = explode("/",$folder);
        $p_id = 0;
        //geting parentId
        $addPrefix = false;
        $i = 0;
        foreach ($e_f as $f){
            if (!empty($f)){
                if ($addPrefix){
                    $f = "save_search::" . $f;
                }
               if ($f === "savedsearch"){
                    $addPrefix = true;
               }
                $sql = "SELECT id FROM foldertree WHERE NAME='$f' AND parentid=$p_id";
                $res = sqlQuery($connection, $sql);
                if ($res)
                {
                    $r = $res->fetch_object();
                    if (!empty($r->id))
                        $p_id = $r->id;
                }
            }
            $i++;
        }
        //get parent name
        $name = $e_f[$i-2];
        $e = array();
        if ($f === "save_search::Dynamic" || $f === "save_search::Static"){
            //geting lastsearch from search (contains last search) - we did a search by clicking dynamic folder
            
            $sql = "SELECT id FROM savedsearch WHERE name='$name' AND username='$username'";
            $res = sqlQuery($connection, $sql);
            $id = $res->fetch_object()->id;
            
            $sql = "SELECT count(docid) as c FROM savedsearchid WHERE id=$id";
            $res = sqlQuery($connection, $sql);
            if($r = $res->fetch_object()){
                
                $total = $res->c;
                write_to_log("TRACE", "nextDoc Total of savedSearch (static or dynamic) - $total");
                
            }
            
            $sql = "SELECT docid FROM savedsearchid WHERE id=$id LIMIT $total";
            
            $res = sqlQuery($connection, $sql);
            if ($res){
                while ($r = $res->fetch_object()){
                    $e[] = $r->docid;
                }
                $k = 0;
            }
        }
        else{    
            //counts total in folder
            $sql = "SELECT count(id) as c FROM files WHERE parentfolderid=$p_id";
            $res = sqlQuery($connection, $sql);
            if ($r = $res->fetch_object()){
                $total = $r->c;
                write_to_log("TRACE", "nextDoc Total of folder - $total");
                
            }
            
            //fill up the array
            $sql = "SELECT id FROM files WHERE parentfolderid=$p_id LIMIT $total";
            
            $res = sqlQuery($connection, $sql);
            while ($r = $res->fetch_object()){
                $e[] = $r->id;
            }
            $k = 0;
        }
    }else{
        //////////////////////////////////
        //
        //	B - get next doc from last search
        //
        //////////////////////////////////
        $sql = "SELECT id FROM savedsearch WHERE username='$username' AND id = ".getLastSavedSearchId($connection, $username);
        $res = sqlQuery($connection, $sql);
        $id = $res->fetch_object()->id;
        
        //gets the total number of documents
        $sql = "SELECT count(docid) as c FROM savedsearchid WHERE id=$id";
        $res = sqlQuery($connection, $sql);
        if($res && $r = $res->fetch_object()){
            $total = $r->c;            
        }
        $sql = "SELECT docid FROM savedsearchid WHERE id=$id LIMIT $total";
        
        $res = sqlQuery($connection, $sql);
        if ($res){
            while ($r = $res->fetch_object()){
                $e[] = $r->docid;
            }
            $k = 0;
        }
    }

   //*
   //* Mutual part of the flow (A and B)
   //*
    if (is_array($e) && sizeof($e) > 1 && in_array($docId, $e))
    
    {
        if ($k === FALSE ){
            //do nothing
        }else{
            $notFound = true;
            $k++;
            if ($k < sizeof($e)){
                $reviewed = array();
                $s = array_slice($e,$k);
                $id = implode(",", $s);
                $sql = "SELECT docId FROM filetags WHERE name='Reviewed' AND docId IN($id) AND userName='$username'";
                $res = sqlQuery($connection, $sql);
                if ($res){
                    while (($r = $res->fetch_object()) && $notFound){
                        $reviewed[$r->docId] = $r->docId;
                    }
                }
                $i = 0;
                $org_pos = array_search($docId, $e);
                $next_pos = $org_pos+1;
                $next_doc = isset($e[$next_pos]) ? $e[$next_pos] : null;
                if(!empty($next_doc)){
                    $ldoc = isLocked($connection, $next_doc);
                    if ($ldoc["isLocked"] === false && $docId !== $next_doc){
                        $res_doc = $next_doc;

                    }
                }

            }
        }
    }
    if(empty($res_doc)){
        return "next doc fail";
    }
    write_to_log("PERMORMENCE", "End nextDoc - $docId" . date('H:i:s'.substr((string)microtime(), 1, 8)));
    
    return array("docId" => $res_doc, "index" => $next_pos,"total" => $total);
}
function priorDoc($mysqli, $docId){
    $username = $_SESSION['username'];
    $sql = "SELECT lastSearch FROM search WHERE userName='$username'";
    $res = sqlQuery($mysqli, $sql);
    if ($res)
    {
        $search = $res->fetch_object();
        $e = explode(",",$search->lastSearch);
        if (sizeof($e) > 1)
        {
            $k = array_search($docId,$e);
            if ($k === FALSE ){
                //do nothing
            }else{
                $notFound = true;
                $k--;
                while ($k >= 0 && $notFound){
                    $newdocId = $e[$k];
                    $sql = "SELECT name FROM filetags WHERE name='Reviewed' AND docId=$newdocId";
                    $res = sqlQuery($mysqli, $sql);
                    if ($res){
                        $r = $res->fetch_object();
                        if ($r->name === 'Reviewed'){
                            $notFound = true;
                        }else{
                            $notFound = false;
                            $docId = $newdocId;
                        }
                    }
                    $k--;
                }
            }
        }
    }
    return array("docId" => $docId);
}

function getTagModelFN()
{
    global $metadata_path;
    return $metadata_path . "/tagModel.js";
}
function saveTagModel($mysqli, $tagModel)
{
    require_once 'sqlUtils.php'; 
    global $username;
    if (empty($tagModel)){
        return array("ERROR" =>"Empty tag model");
    }
        
        $arr = json_decode($tagModel);
        $sql = "DELETE FROM filetags WHERE docId=0;";
        foreach ($arr as $tagGroup){
            $enter = true;
            foreach ($tagGroup->tags as $tag){
                if ($tagGroup->exclusive == "true")
                    $exclusive = 1;
                    else
                        $exclusive = 0;
                        $sql .= "INSERT INTO filetags(docId, exclusive, groupName, name) VALUES (0, $exclusive, \"$tagGroup->name\", \"$tag\");";
                        $enter = false;
            }
            //Emtry Tag with name Only
            if ($enter){
                if ($tagGroup->exclusive == "true")
                    $exclusive = 1;
                    else
                        $exclusive = 0;
                        $sql .= "INSERT INTO filetags(docId, exclusive, groupName, name) VALUES (0, $exclusive, \"$tagGroup->name\", \"\");";
            }
        }
        $result = sqlMultiQuery($mysqli, $sql);
        if ($result === false)
            return array("ERROR" =>"Probelm with insert to DB");
            else
                return array("saveStatus" =>"OK");
                //file_put_contents(getTagModelFN(), $tagModel);
                //return array("saveStatus" =>"OK");
}

function getTagModel($mysqli)
{
    require_once 'sqlUtils.php';
     global $username;
     $res = sqlQuery($mysqli, "SELECT DISTINCT groupName,exclusive FROM filetags where docId=0");
     $response = array();
     if ($res != false){
         while ($row_group = $res->fetch_object()){
             $res_group = sqlQuery($mysqli, "SELECT name FROM filetags WHERE groupName=\"$row_group->groupName\" AND docId=0");
             $arr = array();
             $arr_name = array();
             while ($row_name = $res_group->fetch_object()){
                 $arr_name[] = $row_name->name;
             }
             $arr["name"] = $row_group->groupName;
             if ($row_group->exclusive == "0")
                 $arr["exclusive"] = "false";
                 else
                     $arr["exclusive"] = "true";
                     $arr["tags"] = $arr_name;
                     $response[] = $arr;
         }
     }
     
     return $response;
}

function getTagModel_v2($mysqli, $docid = null)
{
    require_once 'sqlUtils.php';
    global $username, $system_tags;
    $res = sqlQuery($mysqli, "SELECT DISTINCT groupName,exclusive FROM filetags where docId=0");
    if(!empty($docid)){
        $doc_tags = array();
        $res_doc = sqlQuery($mysqli, "SELECT * FROM filetags where docId=$docid");
        while (($row = $res_doc->fetch_object()) != false){
            $doc_tags[$row->groupName][] = $row->name;
        }
    }
    $response = array();
    if ($res != false){
        while (($row_group = $res->fetch_object()) != false){
            $exclusive = $row_group->exclusive == "0" ? false : true;
            $tagGroup = $row_group->groupName;
            if(in_array($tagGroup, $system_tags)){
                continue;
            }
            $res_group = sqlQuery($mysqli, "SELECT name FROM filetags WHERE groupName='$tagGroup' AND docId=0");
            $element_arr = array("id"=>$tagGroup, "name"=>$tagGroup, "exclusive"=>$exclusive, "children"=>array());
            while (($row_name = $res_group->fetch_object()) != false){
                $checked = false;
                $tag_name = $row_name->name;
                if(!empty($docid) && isset($doc_tags[$tagGroup])){
                    if(in_array($tag_name, $doc_tags[$tagGroup])){
                        $checked = true;
                    }
                }
                $element_arr['children'][] = array("id"=>$tag_name, "name"=>$tag_name, "checked"=> $checked);
            }
            $response[] = $element_arr;
        }
    }
    
    return $response;
}


function getSavedSearch($mysqli, $avoid_temp_ss = false, $name = null)
{
    require_once 'sqlUtils.php';
    global $username, $hide_saved_search;
    if(array_key_exists("custom", $_REQUEST)){
        $hide_saved_search[] = "Last search";
    }
    $avoid_temp_ss = $avoid_temp_ss ? " NAME NOT LIKE 'lastsearch%'" : " TRUE ";
    $name_sql = !empty($name) ? " name = \"$name\"" : " TRUE "; 
    $arr = array();
    $res = sqlQuery($mysqli, "SELECT id,name, label,parameters FROM savedsearch WHERE $name_sql  AND username='$username' AND name IS NOT NULL 
       AND $avoid_temp_ss");
    $response = array();
    if ($res != false){
        while ($row = $res->fetch_object()){
            $hide = false;
            $arr["name"] = $row->name;
            $arr["desc"] = (empty($row->label)) ? "" : $row->label;
            $arr["id"] = $row->id; 
            $arr["parameters"] = $row->parameters;
            
            foreach($hide_saved_search as $hid_inst){
                if (strpos($arr["name"], $hid_inst) !== false) {
                    $hide = true;
                    break;
                }
            }
            if(!$hide){
                $response[] = $arr;       
            }
        }
    }
    return $response;
}


function compareByPersonEmail($a, $b)
{
    $aStr = $a? $a.name? a.name : a.email : null;
    $bStr = $b? $b.name? b.name : b.email : null;
    return strcmp($aStr, $bStr);
}
function listItems($mysqli, $request)
{
    require_once 'sqlUtils.php'; 
    $response = array();
    global $userdetails;
    global $username;
    if (!getUserPermission(CAN_SHOW_ALL_USER))
        $condition = "WHERE username='$username'";
        
        $outputFormat = $request["outputFormat"];
        
        $sql = "files";
        $conditionArray = array();
        for ($i = 0; $i < 10; $i++)
        {
            $suffix = $i? $i : "";
            $searchFields = $request["searchFields".$suffix];
            $pattern = $request["searchText".$suffix];
            if (!$searchFields && !$pattern)
                continue;
                
                $conditionArray[$i]["pattern"] = $pattern;
                $conditionArray[$i]["fields"] = $fields = explode(",", $searchFields);
                
                $condition = "";
                if ($pattern)
                {
                    $patternSql = $mysqli->real_escape_string($pattern);
                    if (empty($condition))
                        $condition = "WHERE ";
                        else
                            $condition .= " AND ";
                            
                            $glue = "";
                            foreach ($fields as $field)
                            {
                                $condition .= "$glue$field LIKE '%$patternSql%'";
                                if (empty($glue))
                                    $glue = " OR ";
                            }
                }
                $sql = "(SELECT * FROM ($sql) $condition) as search$i";
        }
        $res = sqlQuery($mysqli, "SELECT * FROM $sql ORDER BY scan_time DESC LIMIT 1000");
        if (!$res)
            return null;
            
            if ($outputFormat == "PersonEmail")
                return resultAsPersonEmail($res, $conditionArray);
                else
                {
                    while ($row = $res->fetch_object())
                        $response[] = $row;
                        
                        return $response;
                }
}
function resultAsPersonEmail($res, $conditionArray)
{
    $foundNames = array();
    $foundNameEmails = array();
    $foundEmails = array();
    while ($res && $row = $res->fetch_object())
    {
        foreach ($conditionArray as $cond)
        {
            foreach ($cond["fields"] as $field)
            {
                $value = $row->$field;
                if (empty($value)) continue;
                
                foreach (preg_split("/\\s*[,;]\\s*/", $value) as $personEmail)
                {
                    if ($cond["pattern"] && strpos($personEmail, $cond["pattern"]) === FALSE)
                        continue;
                        
                        $personAndEmail = explode("<", $personEmail, 2);
                        $person = trim(str_replace(array(">", '"'), "", $personAndEmail[0]));
                        $email = null;
                        if (strpos($person, "@"))
                        {
                            $email = $person;
                            $person = null;
                        }
                        if (count($personAndEmail) == 2)
                            $email = trim(str_replace(array(">", '"'), "", $personAndEmail[1]));
                            
                            $newName = $person && !array_key_exists($person, $foundNames);
                            $newEmail = $email  && !array_key_exists($email, $foundEmails);
                            if ($newName || $newEmail)
                            {
                                $nameAndEmail = array("name" => $person, "email" => $email);
                                if ($newName)
                                {
                                    $foundNames[$person] = $nameAndEmail;
                                    if ($email)
                                        $foundNameEmails[$email] = $nameAndEmail;
                                }
                                else
                                    $foundEmails[$email] = $nameAndEmail;
                            }
                }
            }
        }
    }
    $foundNamesAndEmails = array_merge($foundNames, $foundEmails);
    if(empty($foundEmails)){
        return [];
    }
    ksort($foundNamesAndEmails);
    return array_values($foundNamesAndEmails);
}

function caseStatistics($mysqli, $from_background = false, $savedSearch = null, $username = null, $isAdmin = null)
{
    require_once 'asyncRequst.php';
    require_once 'sqlUtils.php';
    require_once 'parseUtils.php';
    session_write_close();
    
    $username = $username ?? $_SESSION['username'];
    $stats = CacheEng::getFromCache($mysqli,  $username, 'key_stats');
    
    if(!empty($savedSearch) || ($from_background || empty($stats))){
        $response = getStatistic($mysqli, $username,$isAdmin, $from_background, $savedSearch);
    }
    else{
        $response = json_decode($stats->results, true);    
    }

    //calc savedsearch and addd it (or replace previous one from cache)
    $savedSearch =  getDashboardSavedSearch($mysqli, $username);
    
    $response['savedsearch'] = $savedSearch['savedsearch'] ?? "";
    
    return $response;
}

function getOverviewSentiment($mysqli, $username, $lastSearch){
    $results = [];
    $sql = "SELECT  AVG(pro) AS pro, AVG(anti) AS anti , AVG(pro)-AVG(anti) AS mood, AVG(e_concept) AS emotion FROM `emotion` WHERE $lastSearch";
    $res = sqlQuery($mysqli, $sql);
    if(!empty($res)){
        $row = $res->fetch_object();
        if(!empty($row)){
            $results =  array("pro"=>doubleval($row->pro), "anti"=>doubleval($row->anti), "mood" => doubleval($row->mood), "emotion" => doubleval($row->emotion));
        }
    }
    return $results;
    
}

function searchInstanceData($mysqli, $request)
{
    require_once 'sqlUtils.php';
    require_once 'parseUtils.php';
    global $mysqli;
    $dataType = $request["dataType"];
    $searchtext = $request["searchText"];
    //copied from SearchEntity(...)
    $initialTableIndex = getInitialTableIndex($mysqli);
    $tableIndex = $initialTableIndex+1;
    $tableName = $request["tableName"];
    if (!$tableName) $tableName = "person_object";
    $sentiment = $request["sentiment"];
    if ($sentiment === "All"){
        $condition = "$tableName.positive IS NOT NULL OR $tableName.negative IS NOT NULL";
    }elseif ($sentiment === "Positive"){
        $condition = "$tableName.positive IS NOT NULL";
    }elseif ($sentiment === "Negative"){
        $condition = "$tableName.negative IS NOT NULL";
    }
    else{
        $condition = "";
    }
    if (!empty($condition)){
        $condition = "(" . str_replace($tableName, "%PO%", $condition) . ") AND ";
    }
    $megaTable = "person;email";
    $megaCondition = "";
    if (strpos($dataType, "entityAuthors") === 0)
    {
        $megaTable = $tableName;
        if (!empty($condition))
            $megaCondition = $condition;
            
            $tableName = "person_object";
            $condition = "";
    }
    
    $megaSqls = generateSqlStatements($mysqli, $searchtext, $megaTable, $megaCondition, $tableIndex);
    if ($dataType == "entityAuthors")
    {
        $finalSqls = sqlDocsAuthors($megaSqls, $tableIndex);
    }
    else if ($dataType == "authorEntities")
    {
        $authorDocs = sqlPersonAsAuthor($megaSqls, $tableIndex);
        $finalSqls = sqlDocsEntities($authorDocs, $tableName, $condition, $tableIndex);
    }
    else if ($dataType == "recipientAuthors")
    {
        $recipientSql = sqlPersonAsRecipient($megaSqls, $tableIndex);
        $finalSqls = sqlDocsAuthors($recipientSql, $tableIndex);
    }
    else if ($dataType == "authorRecipients")
    {
        $authorDocs = sqlPersonAsAuthor($megaSqls, $tableIndex);
        $finalSqls = sqlDocsRecipients($authorDocs, $tableIndex);
    }
    else return array("error" => "Unsupported data type");
    
    $results = retrieveSearchResults($mysqli, $finalSqls, $initialTableIndex);
    foreach ($results[0]["Rules"]["dummy rule id"] as $result){
        $response[] = $result["instance_name"];
    }
    return $response;
}

function saveRedactedDoc($mysqli, $request)
{
    require_once 'parseUtils.php';
    global $redactedDocsPath;
    $dcid = intval($request["dcid"]);
    $pageCount = intval($request["pageCount"]);
    if (empty($dcid) || empty($pageCount)){
        return array("error" => "Missing document id or page count");
    }
        
    if (!canContribute($mysqli, $dcid)){
        return array("error", "User is not authorized to redact this document");
    }
    
    $redactedId = docToRedactedId($mysqli, $dcid, $request["pageNum"], $pageCount);
if ($redactedId == NULL){
    return array("error" => "Failed to fetch redacted id");
}
    $subfolderName = redactedIdToFolderName($redactedId);
    $fileName = redactedIdToFileName($redactedId);
    $textContent = $request["textContent"];
    if ($textContent)
    {
        $textDir = "$redactedDocsPath/TEXT/$subfolderName";
        mkdir_full($textDir, 0777, true);
        
        file_put_contents("$textDir/$fileName.TXT", $textContent);
    }
    $imageContent = $request["imageContent"];
    if (!$imageContent)
    {
        $imageContent = str_replace(" ","+",$request["imageContentBase64"]);//str_replace("%body%", "black", "<body text='%body%'>");
        if (strpos($imageContent, "data:image/png;base64,") === 0)
            $imageContent = substr($imageContent, strlen("data:image/png;base64,"));
            
            $imageContent = base64_decode($imageContent);
    }
    
    
    if ($imageContent)
    {
        //first, save PNG in metadata - for editing
        $pngFileName = getRedactedImageName($redactedId);
        file_put_contents($pngFileName, $imageContent);
        //then save as TIFF in redacted docs folder
        //TODO: convert PNG to TIFF
        $tiffFileName = getRedactedExportedTiffName($redactedId);
        global $imageMagickPath;
        $failure = executeCommand($imageMagickPath, "convert \"$pngFileName\" \"$tiffFileName\"");
        if ($failure)
            return array("error", "Failed to convert PNG to TIF");
    }
    return $json_return['status'] = "ok";
}

function deleteRedactedPage($redactedId)
{
    //first, save PNG in metadata - for editing
    $imageName = getRedactedImageName($redactedId);
    if (file_exists($imageName)){
        unlink($imageName);
    }
    $tiffFileName = getRedactedExportedTiffName($redactedId);
    if (file_exists($tiffFileName)){
        unlink($tiffFileName);
    }
}

function removeDocFromRedaction($mysqli, $docId)
{
    require_once 'sqlUtils.php';
    $docId = intval($docId);
    if (!canContribute($mysqli, $docId))
    {
        $json_return['error'] = "User is not authorized to contribute to this document";
        return $json_return;
    }
    require_once 'parseUtils.php';
    $redactedRange = docToRedactedRange($mysqli, $docId, NULL, false);
    if (!$redactedRange)
    {
        $json_return['error'] = "Document is not redacted";
        return $json_return;
    }
    
    $redactedId = $redactedRange["redactedId"];
    $redactedEndId = $redactedRange["redactedEndId"];
    for ($redactedPageId = $redactedId; $redactedPageId <= $redactedEndId; $redactedPageId++)
    {
        deleteRedactedPage($redactedPageId);
    }
    //mark doc as unredacted in DB
    sqlQuery($mysqli, "UPDATE files SET redactedId = NULL, redactedEndId = NULL WHERE id = $docId");
    
    $json_return['status'] = "ok";
    return $json_return;
}

function deleteRedactedDoc($mysqli, $request)
{
    $docId = intval($request["dcid"]);
    $pageNum = intval($request["pageNum"]);
    if (!canContribute($mysqli, $docId))
    {
        $json_return['error'] = "User is not authorized to contribute to this document";
        return $json_return;
    }
    require_once 'parseUtils.php';
    $redactedId = docToRedactedId($mysqli, $docId, $pageNum, false);
    deleteRedactedPage($redactedId);
    $json_return['status'] = "ok";
    return $json_return;
}

function allGraphsList($mysqli){
    require_once 'sqlUtils.php';
    global $mysqli;
    
    $query = "SELECT name, type FROM dashboard";
    $result = sqlQuery($mysqli, $query);
    while ($row = $result->fetch_object()) {
        $response[] = array("name"=>$row->name, "type"=>$row->type);
    }
    return $response;
}

function statisticsList($mysqli)
{
    require_once 'sqlUtils.php';
  
    global $username;
    $response = array();
    $sql = "SELECT * FROM dashboard WHERE x_point is NOT NULL AND y_point is NOT NULL AND username=\"$username\" ORDER BY y_point, x_point";
    $res = sqlQuery($mysqli, $sql);
    
    if ($res != false){
        while ($row = $res->fetch_object()){
            $arr[$row->y_point][$row->x_point] = $row;
        }
        if(empty($arr)){
            write_to_log("ERROR"," empty results from dashbord");
            return  "empty results from dashbord";
        }
        $response[] = array();
        $i = 0;
        foreach ($arr as $key => $val){
            
            $j = 0;
            while ($key != $i){
                $response[$i++] = array();
            }
            foreach ($val as $x){
                $details = json_decode($x->details,true);
                $response[$i][$j]["name"] = $x->name;
                $response[$i][$j]["type"] = $x->type;
                $response[$i][$j]["searchField"] = $x->search_field;
                $response[$i][$j]["slices"] = $x->slices;
                $response[$i][$j]["sort_by"] = $details["sort_by"];
                $response[$i][$j]["display_type"] = $details["display_type"];
                $response[$i][$j]["slices"] = $x->slices;
                $response[$i][$j]["refreshPeriod"] = $x->refresh_seconds;
                $j++;
            }
            $i++;
        }
    }
    return $response;
}

function updateStatisticsList($mysqli){
    require_once 'sqlUtils.php';
    global $username;
    
    $query = "UPDATE dashboard
				SET x_point=NULL, y_point=NULL, slices=NULL
				WHERE x_point IS NOT NULL AND username=\"$username\";";
    
    $graphsArray = json_decode($_REQUEST["graphs"]);
    $y = 0;
    $continue = false;
    $queryloop="";
    $details_sql= "";
    foreach ($graphsArray as $graphs){
        $x = 0;
        foreach ($graphs as $graph){
            if (strlen($graph->slices) > 0){
                $slice = ",slices="  . $graph->slices;
            }
            else{
                $slice = "";
            }
            
            if (strlen($graph->display_type) > 0){
                $display_type =  $graph->display_type;
                $details_sql .= "UPDATE dashboard SET details = JSON_SET(details, '$.display_type', $display_type) WHERE NAME='$graph->name' AND username='$username';";
            }
            
            if (strlen($graph->sort_by) > 0){
                $sort_by = $graph->sort_by;
                $details_sql .= "UPDATE dashboard SET details = JSON_SET(details, '$.sort_by', $sort_by) WHERE NAME='$graph->name' AND username='$username';";
            }
            
            if((is_object($graph)) && !is_null($y) && !is_null($x)){
                $queryloop .= "UPDATE dashboard SET y_point=$y, x_point=$x $slice WHERE NAME='$graph->name' AND username='$username';";
                $x++;
            }
        }
        $y++;
        $continue = true;
    }
    if(!empty($details_sql)){
        $queryloop .= $details_sql;
    }
    
    if ($continue === true && !empty($queryloop) ){
        sqlMultiQuery($mysqli, $query.$queryloop);
    } else{
        $response[status] = false;
    }
    $err = $mysqli->error;
    if (strlen($err) > 1){
        $response['status'] = false;
    }else {
        $response['status'] = true;
    }
    return $response;
}

function getTrendData($mysqli) {
    require_once 'parseUtils.php';
    global $username;
    $return = getTrendInfo($username, $mysqli, $_GET["name"]);
    return $return;
}


function getPieData($mysqli, $v2){
     require_once 'sqlUtils.php'; 
     require_once 'parseUtils.php'; 
     global $username;
     $isOrgText = getSystemSettingsProp($mysqli, "isOrgText");
     
     $pieInfo = json_decode(getPieInfo($mysqli, $v2, $username, $_GET["name"], $_REQUEST['savedsearch'] ?? null), true);
//      $pieInfoRes = handeleSlices($_GET["slices"],$pieInfo);
     $pieInfo["isOrgText"] = $isOrgText;
     
     return $pieInfo;
 }
 
 function handeleSlices($slices,$pieInfo) {
     if (!is_numeric($slices)) {
         $slices = 10;
     }
     if ($pieInfo && isset($pieInfo["amount"])) {
         $pieInfo["amount"] = array_slice($pieInfo["amount"], 0, $slices);
     }
     return $pieInfo;
 }
 
 function saveSearchToFiles($mysqli, $name = null){
     global $username;
     $searchName = $name ?? $_REQUEST["name"];
     if(empty($searchName)){
         return ["error"=>"no saved search param"]; 
     }
     
     
     $searchName = $mysqli->real_escape_string($searchName);
     //check if search exsist
     $res = sqlQuery($mysqli, "SELECT id FROM foldertree WHERE Name = 'save_search::$searchName' AND username='$username'");
     if (!$res){
         $json_return['status'] = "Probelm to get foldertree";
         return $json_return;
     }
     
     if(!empty($res->fetch_object()->id)){
         $json_return['status'] = "In use";
         return $json_return;
     }
     
     $res = sqlQuery($mysqli, "SELECT id FROM foldertree WHERE Name = 'savedsearch' AND username='$username'");
     
     if (!$res || !$res->num_rows){
         die("couldn't get/create 'saved searches' folder");
     }
         
     $savedSearchesId = $res->fetch_object()->id;
     
     $res = sqlQuery($mysqli, "INSERT INTO foldertree(Name, ParentId, username) VALUES('save_search::$searchName', $savedSearchesId, '$username')");
     
     $res = sqlQuery($mysqli, "SELECT id FROM foldertree WHERE Name = 'save_search::$searchName' AND username='$username'");
     $newSearchId = $res->fetch_object()->id;
     
     $res = sqlQuery($mysqli, "INSERT INTO foldertree(Name, ParentId, username) VALUES('save_search::Dynamic', $newSearchId, '$username')");
     $res = sqlQuery($mysqli, "INSERT INTO foldertree(Name, ParentId, username) VALUES('save_search::Static', $newSearchId, '$username')");
     
     //save to savesearch table
     
     $res = sqlQuery($mysqli, "SELECT parameters,search_values, id,per,org,loc FROM savedsearch WHERE username='$username' AND id =".getLastSavedSearchId($mysqli, $username));
     $obj = $res->fetch_object();
     $parameters = $mysqli->real_escape_string($obj->parameters);
     $search_values = $mysqli->real_escape_string($obj->search_values);
     
     $oldSearchId = $obj->id;
     $org = $mysqli->real_escape_string($obj->org ?? "");
     $loc = $mysqli->real_escape_string($obj->loc ?? "");
     $per = $mysqli->real_escape_string($obj->per ?? "");
     $date = date('YmdHis');
     $res = sqlQuery($mysqli, "INSERT INTO savedsearch(name, parameters, search_values, username, date, org, loc, per) VALUES('$searchName', '$parameters', '$search_values', '$username', '$date', '$org', '$loc', '$per')");
     
     //get savesearch id and replace id on save searchid table
     $res = sqlQuery($mysqli, "SELECT id FROM savedsearch WHERE NAME='$searchName' AND username='$username'");
     $newSearchId = $res->fetch_object()->id;
     $res = sqlQuery($mysqli,"UPDATE savedsearchid SET id=$newSearchId WHERE id=$oldSearchId");
     
     $json_return['status'] = "Ok";
     return $json_return;
 }
 
 
 function saveTrendData($mysqli){
     require_once 'parseUtils.php';
     require_once 'sqlUtils.php';  
     global $username;
     global $dashboardTemplateUsername;
     global $defaultDashboradNames;
     
     $names_arr = fetchFromDefaultUser($mysqli, $dashboardTemplateUsername, $username);
     $default_user_names = $names_arr['names'];
     $merged_array = array_unique(array_merge(array_keys($default_user_names), array_keys($defaultDashboradNames)));
     
     $trendName = $_REQUEST["name"];
     
     //check if we have the new db for the dashboard table
     $the_return_value = null;
     $query = "SELECT name,x_point FROM dashboard WHERE name='$trendName' AND username=\"$username\"";
     $res = sqlQuery($mysqli, $query);
     $res = $res->fetch_object();
     if ($res === false) {
         $json_return['status'] = "Probelm to create new graph";
         write_to_log("ERROR", "Probelm to create new graph query=$query");
         $the_return_value = $json_return;
     } elseif (strlen($res->name) > 1 && $res->x_point != NULL) {
         $json_return['status'] = "In use";
         $the_return_value = $json_return;
     }
     elseif(in_array(strtolower($trendName), array_map('strtolower', $merged_array))){
         $json_return['status'] = "In use of a default element";
         $the_return_value = $json_return;
     }
     else {
         $isPie = $_REQUEST["isPie"];
         $trendType = $_REQUEST["trendType"];
         if (!$trendType) {
             $trendType = ($isPie) ? "pie" : "graph";
         }
         
         $slices = $_REQUEST["slices"];
         $table = $_REQUEST["table"];
         
         $data = array(
             "pastPeriod" => $_REQUEST["pastPeriod"],
             "fromDay" => $_REQUEST["fromDay"],
             "fromMonth" => $_REQUEST["fromMonth"],
             "fromYear" => $_REQUEST["fromYear"],
             "toDay" => $_REQUEST["toDay"],
             "toMonth" => $_REQUEST["toMonth"],
             "toYear" => $_REQUEST["toYear"],
             "freeTextInput" => $_REQUEST["freeTextInput"],
             "entitiesSelection" => $_REQUEST["entitiesSelection"],
             "columnNum" => $_REQUEST["columnNum"],
             "asAvg" => $_REQUEST["asAvg"],
             "uniqueTable" => $_REQUEST["uniqueTable"],
             "savedsearch" => $_REQUEST["savedsearch"],
             "isTotalMentionsHidden" => $_REQUEST["isTotalMentionsHidden"],
             "isDateRangeLimitedToMentions" => $_REQUEST["isDateRangeLimitedToMentions"],
             "lineGraphOptions" => $_REQUEST["lineGraphOptions"],
             "barGraphOptions" => $_REQUEST["barGraphOptions"],
             "name" => $trendName,
             "username" => $username,
             "display_type" => "pie",
             "sort_by" => "mentions"
         );
         if ($trendType === "pie" || $trendType === "bubbles") {
             $data["table"] = $table;
             $conceptText = strtolower($_REQUEST["freeTextInput"]);
             $concept_table = explode(";", $conceptText)[0];
             $searchField = "searchOther=$concept_table";
             
             $def_arr = array("searchEvents" => "event", "searchOrganization" => "organizational_identity",
                 "searchPerson" => "person_object", "searchPlaces" => "place_object", "file_format" => "file_format","abstract_idea_entity"=>"abstract_idea_entity");
             global $dropdownArr;
             $fields_arr = array_keys($dropdownArr);
             
             if (in_array($table, $def_arr)) {
                 
                 if ($_REQUEST['hasSentiment']) {
                     
                     $searchField = array_search($table, $def_arr);
                     $name = $_REQUEST['freeTextInput'];
                     $searchField = "searchSentiment=^$table^$name";
                     write_to_log("TRACE", $searchField);
                 } else {
                     $searchField = array_search($table, $def_arr);
                 }
             }
             
             
             if (in_array($table, $fields_arr))
                 $searchField = $table;
                 
                 // We currently only support 5 slices or the default 10 slices
                 // If anything other than 5 is selected, we revert to the default 10.
                 if ($slices != 5)
                     $slices = 10;
                     
                     $data["slices"] = $slices;
                     $data["searchField"] = $searchField;
         }
         elseif ($trendType === "network") {
             //clean array
             $data = array();
             $data["relationType"] = $_REQUEST["relationType"];
             $data["subjectType"] = $_REQUEST["subjectType"];
             $data["objectType"] = $_REQUEST["objectType"];
             $data["groupBy"] = $_REQUEST["groupBy"];
             $data["savedsearch"] = $_REQUEST["savedsearch"];
             $data["slices"] = $slices;
             $data["subjectName"] = $_REQUEST["subjectName"];
             $data["objectName"] = $_REQUEST["objectName"];
             $data["ignoreSubject"] = $_REQUEST["ignoreSubject"];
             $data["type"] = $_REQUEST["type"];
             $data["name"] = $_REQUEST["name"];
             $searchField = "";
         }
         $data["trendType"] = $trendType;
         
         $graph_details = json_encode($data);
         
         $query = "SELECT MAX(x_point) as x FROM dashboard";
         $res = sqlQuery($mysqli, $query);
         if ($res != false) {
             if ($row = $res->fetch_object())
                 $x = 1 + $row->x;
         }
         $refresh_seconds = ",600";
         if ($_REQUEST["pastPeriod"]) {
             if ($_REQUEST["pastPeriod"] == 1)
                 $refresh_seconds == ",60";
                 elseif ($_REQUEST["pastPeriod"] == 2)
                 $refresh_seconds == ",120";
                 elseif ($_REQUEST["pastPeriod"] <= 7)
                 $refresh_seconds == ",300";
         }
         //if exsist delete before insert
         $query = "DELETE FROM dashboard WHERE username=\"$username\" AND name='$trendName'";
         $res = sqlQuery($mysqli, $query);
         
         if ($trendType === "pie" || $trendType === "bubbles" || $trendType === "network") {
             $query = "INSERT INTO dashboard (x_point,y_point,name,type,search_field,username,refresh_seconds,slices)
			VALUES ($x, 0, \"$trendName\", \"$trendType\", \"$searchField\", \"$username\"$refresh_seconds, $slices)";
         } else {
             $query = "INSERT INTO dashboard (x_point,y_point,name,type,username,refresh_seconds)
		VALUES ($x,0, \"$trendName\", \"trend\", \"$username\"$refresh_seconds)";
         }
         
         
         
         
         $res = sqlQuery($mysqli, $query);
         
         //we dont need to put data in the results column (it was created empty in the other case)
         setTableElementData($mysqli, 'dashboard',$trendName, 'details', $graph_details, 'username',$username);
         
         $json_return['status'] = "Ok";
         $the_return_value = $json_return;
     }
     
     
     return $the_return_value;
 }
 
 
 function getTags($mysqli, $notReviewed=true){
     require_once 'sqlUtils.php'; 
     $query = "SELECT groupName,name FROM filetags WHERE docId=0";
     $res = sqlQuery($mysqli, $query);
     if ($notReviewed){
         if($_REQUEST["v2"]){
             $arr[] =  NOT_REVIEWED;  
         }
         else{
             $arr[NOT_REVIEWED] =  NOT_REVIEWED;  
         }
         
     }
     while ($obj = $res->fetch_object()){
         $name = $obj->groupName . " => " .  $obj->name;
         if($_REQUEST["v2"]){
             $arr[] = $name;
         }
         else{
             $arr[$name] = $name;
         }
         
     }
     return $arr;
         
         
 }
 
 function getConceptFields($mysqli){
     global $restrictedTables;
     return getTablesExcept($mysqli, $restrictedTables);
 }
 
 function getConceptsCount($mysqli){
     global $restrictedTables;
     return getCocnceptsExcept($mysqli, $restrictedTables);
 }
 
 
 function getTableData($mysqli, $field){
     require_once 'sqlUtils.php';
     
     $query = "SELECT DISTINCT $field AS t FROM files WHERE $field IS NOT NULL";
     
     $res = sqlQuery($mysqli, $query);
     $arr = array();
     while ($obj = $res->fetch_object()){
         $e = explode(";", $obj->t);
         foreach ($e as $v){
             if (strlen($v) > 0)
                 $arr[$v] = $v;
         }
     }
     return $arr;
     
 }
 
 function getDropDownData($mysqli, $table, $v2){
     require_once 'sqlUtils.php';  
     global $aggregationArr, $supported_formats, $hideConceptInstances;
     $arr = array();
//      $arr["None"] = "None";
     if (!empty($aggregationArr) && array_key_exists($table,$aggregationArr))
     {
         foreach ($aggregationArr[$table] as $k => $v){
             $arr[$k] = $k;
             foreach ($v as $val){
                 if($v2){
                 $arr[$val] = $val;
                 }
                 else{
                     $arr[] = $val;
                 }
             }
         }
     }else{
         if (!empty($hideConceptInstances) && array_key_exists($table, $hideConceptInstances)){
             $hiddenInstancesSql = "  value  NOT IN ('" . join("','", $hideConceptInstances[$table]) . "') ";
         }
         else{
             $hiddenInstancesSql = " TRUE ";
         }
         $prefix = $_REQUEST["prefix"] ?? null;
         $prefix_sql = !empty($prefix) ? " value LIKE \"$prefix%\"" : " TRUE "; 
         $sql = "SELECT DISTINCT value FROM $table WHERE $hiddenInstancesSql AND value IS NOT NULL AND $prefix_sql LIMIT 500";
         $res = sqlQuery($mysqli, $sql);
         if (!empty($res)){
             while ($row = $res->fetch_object()){
                 if($v2){
                     $arr[] = $row->value; 
                 }
                 else{
                     $arr[$row->value] = $row->value;          
                 }
             }
         }
         if (!empty($aggregationArr) && array_key_exists($table,$aggregationArr["add_to_table"])){
             foreach ($aggregationArr["add_to_table"][$table] as $k => $v){
                 $arr[$v] = $k;
             }
         }
     }
     if ($table == "languages"){
         unset($arr["Unknown language"]);
     }

     if($table=='file_format'){
         foreach($arr as $ext){
             if(!in_array($ext,$supported_formats)){
                 unset($arr[$ext]);
             }
         }
     }
     return $arr;
 }
 
 function getSlotData($mysqli, $slotdataColumns = null, $slotdatavalues = null){
     require_once 'sqlUtils.php';
     global $slotToCaption;
     $first = true;
     $col_not_null = "";
     $col = "";
     $arr = array();
     foreach ($slotToCaption as $k => $v){
         $e = str_replace("-", "_", $k);
         if ($first){
             $first = false;
             $col .= "$e";
         }else{
             $col .= ",$e";
         }
         $col_not_null .= " AND $e IS NOT NULL";
     }
     $arr = getSlotDataArr($mysqli, $slotdataColumns, $slotdatavalues, $col);
     
      return $arr;
 }
 
 function checkValuesInDb($mysqli, $values, $concept = null){
//      if(!empty($concept)){
//          global $entitiesTypeToTable;
//          $conceptname = strtolower($concept);
//          $conceptname = array_key_exists($conceptname, $entitiesTypeToTable) ? $entitiesTypeToTable[$conceptname] : $conceptname;
//          $concept_sql =  "conceptName= \"$conceptname\"";
//      }
//      else{
//          $concept_sql =  " TRUE ";
//      }
    
     
     $results = [];
     $show_key = false;
     foreach ($values as $col){
         if(is_array($col)){
             $col = $col[0];
         }
         $col = strtolower(str_replace('-', '_', $col));
         if(!TablesInformation::isColumnExists($mysqli, 'slotdata',$col)){  
             continue;
         }
         $results[] = ['name'=> $col, 'exists'=> true];
         
//          $sql = "SELECT DISTINCT $col AS v FROM slotdata WHERE $concept_sql AND $col <> '' LIMIT 1";
//          $res = sqlQuery($mysqli, $sql);
//          if (!empty($res)){
//              $row = $res->fetch_object();
//              $exists = !empty($row->v);
//              $results[] = ['name'=> $col, 'exists'=> $exists];
//              if(!$show_key && $exists){
//                  $show_key = true;
//              }
//          }
         
     }
     return ["values"=>$results,"show_key"=>true];//$show_key];
 }
     
 function getSlotsHirarchy($mysqli){
     global $slotsHirarchy;
     $all_arr = $any_array =  [];
     
     foreach($slotsHirarchy as $concept=>$arr){
        foreach($arr as $key=>$values){
            $res_arr = checkValuesInDb($mysqli, $values, $concept);
            $response[$concept][$key] = ["label"=>$key, "values"=>$res_arr["values"],"show"=>$res_arr["show_key"]];
            $all_arr = array_merge($all_arr, $values);
        }
     }
     
     $res = (array_count_values($all_arr));
     foreach($res as $val=>$count){
         if($count > 1){
             $any_array[$val] =[$val];
         }
     }
    if(!empty($any_array)){
        $any_array = checkValuesInDb($mysqli, $any_array);  
        foreach($any_array["values"] as $arr){
            $response["Any"][$arr["name"]] = ["label"=>$arr["name"], "values"=>[$arr],"show"=>$arr["exists"]];
            
        }
    }
   
     return $response;
 }
 
 function getSlotDataArr($mysqli, $slotdataColumns, $slotdatavalues, $col){
     global $slotToCaption;
  
     $arr = array();
     $temp_arr = array();
     if(empty($slotdataColumns) && empty($slotdatavalues)){
         global $mysqli;
         $id = $_REQUEST["id"] ?? null;
         $name = $_REQUEST["name"] ?? null;
         $query = "SELECT DISTINCT $col FROM slotdata,(SELECT caption AS c FROM slotdata WHERE docId=$id AND conceptUrl='$name') AS s WHERE s.c=slotdata.caption AND docId=$id";
         
         $res = sqlQuery($mysqli, $query);
         
         if (!empty($res)){
             while ($row = $res->fetch_object()){
                 foreach($row as $key => $value){
                     $e = str_replace("_", "-", $key);
                     if (strlen($value) > 0)
                         $arr[$slotToCaption[$e]['label']] = $value;
                 }
             }
         }
     }
     else{
         $slotdataColumns = str_getcsv(trim($slotdataColumns,'()'), ",", '"');
         $slotdatavalues = str_getcsv(trim($slotdatavalues,'()'), ",", '"');
         
         $slotdataColumns = array_map('trim',$slotdataColumns);
         
         $col_arr = explode(',', $col);
         
         $req_col = array_intersect($slotdataColumns,$col_arr);
         
         for($i = 0; $i < count($slotdataColumns); $i++){
             if(in_array($slotdataColumns[$i], $req_col)){
                 $temp_arr[$slotdataColumns[$i]] = str_replace('"', "", trim($slotdatavalues[$i] ?? ""));
             }
         }
         
         foreach($temp_arr as $key=>$value){
             $key = str_replace("_", "-", $key);
             if (strlen($key) > 0)
                 $arr[$slotToCaption[$key]['label']] = $value;
         }
     }
     if (empty($arr)){ 
         $arr['name'] = isset($_REQUEST['title']) ? $_REQUEST['title'] : null;
     }
     return $arr;
 }
 
 function downloadGraph(){
     write_to_log("TRACE", "downloading graph data -> redirecting to download.php");
     //TODO: consider to remove when making a web service
     $_REQUEST["origFile"] = false;
     require_once("download.php");
 }
 
 function getExtraColumns($mysqli, $label_view = false)
 {
     require_once 'sqlUtils.php';
     global $extraColumns, $dropdownArr, $hideFields;
     $extraColumns = array_merge($extraColumns, $dropdownArr);
     $result = [];
     //filter out empty tables (with less than 1 row)
     $sql = "";
     $res = sqlQuery($mysqli, "SHOW TABLES");
     while ($res && $row = $res->fetch_array())
         $tables[$row[0]] = true;
         
         $chunks = [];
         $ind =0;
         $numUnions =0;
         foreach($extraColumns as $table => $name)
         {
             if(!TablesInformation::isTableExists($mysqli, $table)){
                 continue;
             }
             if(!TablesInformation::isColumnExists($mysqli, $table, "value")){
                 continue;
             }
             
             if (!array_key_exists(strtolower($table), $tables) || (array_key_exists($table, $hideFields) && empty($hideFields[$table]))){
                 continue;
             }
                 
                 
             if (!empty($sql))
                 $sql .= "\r\n UNION \r\n";
                 
                 $sql .= "SELECT * FROM (SELECT docId, '$table' AS Tabl, '" . $mysqli->real_escape_string($name) . "' AS NAM FROM $table LIMIT 1) AS tmptbl" . ++$ind;
                 //for memsql - which has a limit (of about 40) on the number of unions - split to chunks of ~20 unions
                 if (++$numUnions > 20)
                 {
                     $chunks[] = $sql;
                     $sql = "";
                     $numUnions = 0;
                 }
         }
         if (!empty($sql))
         {
             $chunks[] = $sql;
             $sql = "";
         }
         
         foreach ($chunks as $sql)
         {
             $res = sqlQuery($mysqli, $sql);
             while ($res && $row = $res->fetch_object()){
                 if($label_view){
                     $result[] = ['label'=>$row->Tabl, 'caption'=>$row->NAM];
                     
                 }
                 else{
                     $result[$row->Tabl] = $row->NAM;
                     
                 }
             }
         }
         return $result;
 }
 
 function getKeywordTables($mysqli)
 {
     require_once 'parseUtils.php';
     global $dropdownArr;
     $t = getTables($mysqli);
//      foreach ($dropdownArr as $k=>$v){
//          $t[$v] = $k;
//      }
     return $t;
 }
 
 
 function getExtraEntityTypesArray()
 {
     global $extraEntityTypesArray;
     
     // Each element's 'key' is unchanged, while its 'value' is converted into
     // an associative array in the form:
     // array(
     // "name" => [old 'key' goes here, i.e. the table name]
     // "name" => [old 'value' goes here, i.e. the displayed name]
     // );
     function convertArrayToAssociativeForm(&$a, &$b){
         $a = array(
             "name" => $a,
             "value" => $b
         );
     };
     
     // First we must convert each element of the array into its own associative array.
     array_walk($extraEntityTypesArray, function(&$a, &$b){
         convertArrayToAssociativeForm($a, $b);
     });
         
         // Now, before we return the array, we discard the unecessary key values of the 'parent array',
         // by converting it from an associative array into an indexed array.
         return(array_values($extraEntityTypesArray));
         
 }
 
 
 
 function getTablesSearch($mysqli, $dropdownArr){
     $tables = array();
     $tables = getTables($mysqli);
     foreach ($dropdownArr as $k=>$v){
         $tables[$v] = $k;
     }
     return $tables;
 }
 
 function getSmartSearch($mysqli, $table, $value){
     require_once 'sqlUtils.php';
     //$parameters["value"] = "hamas^shlomi";
     global $dropdownArr;
     global $relation_exceptions;
     
     //search in manageEntities
     if ($table == "saved")
     {
         $val = $mysqli->real_escape_string($value);
         $sql = "SELECT aggregatedName FROM entitiesmanage WHERE name=\"$val\" AND entity='smart'";
         $res = sqlQuery($mysqli, $sql);
         $conceptArr = array();
         if (!empty($res)){
             $obj = $res->fetch_object();
             $conceptArr = json_decode($obj->aggregatedName, true);
         }
         return $conceptArr;
     }
     $tables = getTablesSearch($mysqli, $dropdownArr);
     
     $e = explode("^",$value);
     $conceptArr = array();
     foreach ($e as $searchConcept){
         if (strlen($searchConcept) > 0){
             if (strlen($table) > 0){
                 //we don't need to find the table
                 $c["concept"][$table] = $table;
                 $c["caption"][$searchConcept] = $searchConcept;
             }else
                 //get analyze word
                 $c = analyzeWord($searchConcept);
                 
                 if (empty($c) || $c == null){
                     $c=array();
                     $c["caption"][$searchConcept] = $searchConcept;
                     $c["concept"] = array("person_object","organizational_identity","place_object","event");
                     write_to_log("ERROR", "getSmartSearch - analyze word return empty ". date('H:i:s'.substr((string)microtime(), 1, 8)));
                 }
                 if (!array_key_exists(strtolower($searchConcept), $c["caption"])){
                     $c["caption"][$searchConcept] = $searchConcept;
                     $c["concept"]["concept"] = "concept";
                 }
                 $val = implode("','", $c["caption"]);
                 foreach ($c["concept"] as $k) {
                     $caption = array();
                     if (in_array($k, $tables)){
                         if ($k == "person_object" || $k == "organizational_identity" || $k == "place_object" || $k == "event" || $k =='abstract_idea_entity'){
                             foreach ($c["caption"] as $c_v){
                                 global $entitiesArray;
                                 $gArr = getMangeEntities($mysqli, $entitiesArray[$k], $c_v, false, true);
                                 if (sizeof($gArr["rows"]) > 0){
                                     $caption[$c_v] = $gArr["rows"];
                                 }
                             }
                         }
                         else{
                             //check if value exsist on table
                             foreach ($c["caption"] as $v){
                                 $val = $mysqli->real_escape_string($v);
                                 
                                 $name = array_key_exists($k, $dropdownArr) ? "value" : 'name';
                                 
                                 $query = $k == "concept" ?
                                 "SELECT DISTINCT docId FROM $k WHERE $name IN ('$val') OR orgText IN ('$val') LIMIT 1":
                                 "SELECT DISTINCT docId FROM $k WHERE $name IN ('$val') LIMIT 1";
                                 
                                 $res = sqlQuery($mysqli, $query);
                                 if (!empty($res)){
                                     $obj = $res->fetch_object();
                                     if (!empty($obj) && $obj->docId > 0){
                                         $caption[$v][] = $v;
                                         //Add related results without general concept
                                         if ($k != "concept" && !array_key_exists(strtolower($k),$relation_exceptions)){
                                             //if the concept is a city - get the city province/state if exists
                                             if($k == "city" || $k=='province'){

                                                 $caption["Related Results"]= handleCityProvince($mysqli, $k, $v, $val);
                                             }
                                             else{
                                                 
                                                 $query = "SELECT DISTINCT name FROM $k ORDER BY name ASC limit 50";
                                                 $res = sqlQuery($mysqli, $query);
                                                 if (!empty($res)){
                                                     while ($obj = $res->fetch_object()){
                                                         if ($obj->name != $val){
                                                             $caption["Related Results"][] = $obj->name;
                                                         }
                                                     }
                                                 }
                                             }
                                         }
                                     }
                                 }
                             }
                         }
                         if (! empty($caption)) {
                             $conceptArr[$searchConcept][$k]["key"] = $k;
                             $conceptArr[$searchConcept][$k]["value"] = $caption; //$value_arr
                             $table_res = array_search($k,$tables);
                             if(empty($table_res)){
                                 $table_res = array_search($k,$tables);
                             }
                             $conceptArr[$searchConcept][$k]["caption"] = tableNameToPretty(array_search($table_res,$tables));
                         }
                     }
                 }
         }
     }
     
     return $conceptArr;
 }

 function handleCityProvince($mysqli, $k, $v, $val){
         $ind_name = "";
         $ind_val = "";
         $caption = array();
         $query = "SELECT * FROM $k WHERE name = '$v'";
         $res = sqlQuery($mysqli, $query);
         if (!empty($res)){
             while ($obj = $res->fetch_object()){
                 $province = $obj->province;
                 $country = $obj->country;
             }
         }
         if(isset($province)){
             $ind_name='province';
             $ind_val=$province;
         }
         else{
             $ind_name = 'country';
             $ind_val=$country;
         }
         $query = "SELECT * FROM (SELECT DISTINCT name,dockey FROM $k WHERE $ind_name='$ind_val' GROUP BY name limit 50) as t ORDER BY t.name ASC ";
         
         $res = sqlQuery($mysqli, $query);
         if (!empty($res)){
             while ($obj = $res->fetch_object()){
                 if ($obj->name != $val){
                        $caption[] = $obj->name;                    
                 }
             }
         }
        
         return $caption;
     }
 
     //OBSELETE
     function applySmartSearch($mysqli, $username, $param, $ignore_limit=null, $numberToFetch=null, $offset=null){
//     $param = "{\"0\":{\"smart\":{\"Steve Mosko\":{\"person_object\":\"Steve Mosko\"}}}}";
         write_to_log("INFO", "$username: search query: $param");
         $res = getDetailsForSmartSearch($mysqli, $param);  
         
         $_REQUEST["search_function"] = true;
         //1 - if it's a download csv with all files (more than 500 limit) action
         if($ignore_limit){
             searchWithoutLimit($mysqli, $res);
         }
         else
         {
             require_once 'searchEng.php';  
             //2 - if it's an numberToFetch offset request search (for scrolling down results)
             if(isset($offset) && isset($numberToFetch)){
                 search($mysqli, null, $res["searchParameters"], $res["entityParmetersArr"], $res["tableAndCaption"], $param, false, $numberToFetch, $offset);
             }
             //3 -  if it's a regular search action
             else{
                 die(search($mysqli, null, $res["searchParameters"], $res["entityParmetersArr"], $res["tableAndCaption"], $param));
         }
     }
 }
 
 function createSqlGetDocIdMultipleSelect($arr,$operator){
     $sql = "SELECT DISTINCT files.id FROM files WHERE files.username LIKE '%%' AND (";
     $f = true;
     foreach ($arr as $v){
         if ($f)
             $f = false;
             else
                 $sql .= " $operator ";
                 $sql .= " files.id IN ($v) ";
     }
     $sql .=")";
     return $sql;
 }
 
 function autoComplete($mysqli, $partName,$tableName){
     require_once 'sqlUtils.php';  
     session_write_close();
     $partName = $mysqli->real_escape_string($partName);
     $query = "SELECT DISTINCT name FROM entitiesmanage WHERE name LIKE('$partName%') LIMIT 10";
     $res = sqlQuery($mysqli, $query);
     $count = 0;
     if ($res){
         while ($obj = $res->fetch_object()){
             $response["saved"][] = $obj->name;
             $count++;
         }
     }
     $limit = 10 - $count;
     if (strlen($tableName) == 0){
         $tableName = "concept";
     }
     //caption search
     if ($limit > 0){
         $query = "SELECT DISTINCT name FROM $tableName WHERE NAME LIKE('$partName%') LIMIT $limit";
         $res = sqlQuery($mysqli, $query);
         if ($res){
             while ($obj = $res->fetch_object())
                 if (!empty($obj)){
                     $response["concept"][] = $obj->name;
                     $count++;
             }
         }
     }
     //org text search
     $limit = 10 - $count;
     if ($limit > 0){
         $query = "SELECT DISTINCT orgText FROM $tableName WHERE orgText LIKE('$partName%') LIMIT $limit";
         $res = sqlQuery($mysqli, $query);
         if ($res){
             while ($obj = $res->fetch_object()){
                 if (!empty($obj)){
                     $response["concept"][] = $obj->orgText;
                     $count++;
                 }
             }
         }
     }
     return $response ?? null;
 }

 
 function fetchMoreSearchResults($mysqli, $username, $v2 = false){
     $param = get_Params($mysqli, $_SESSION['username'], $v2);
     $numberToFetch = $_REQUEST['numberToFetch'];
     $offset = $_REQUEST['offset'];
     applySmartSearch($mysqli, $username, $param,false, $numberToFetch, $offset);
 }

 
 function get_Params($mysqli, $username, $v2 = false){
     $arg = $v2 ? "search_values" : "parameters";
     $res = sqlQuery($mysqli, "SELECT $arg AS params FROM savedsearch WHERE username='$username' AND id =".getLastSavedSearchId($mysqli, $username));
     $obj = $res->fetch_object();
     $parameters = $obj->params;
     return $parameters;
 }
 
 //search action with igonore_limit parameter for downloading the file list in csv format
 function searchWithoutLimit($mysqli, $res){
     require_once 'searchEng.php';
     $time_start = microtime(true);
     search($mysqli, null, $res["searchParameters"], $res["entityParmetersArr"], $res["tableAndCaption"], null,true);
     $time_end = microtime(true);
     $execution_time = ($time_end - $time_start);
     echo " | Finished writing the data in ".number_format($execution_time,3) ." seconds";
 }
 
//API for clickhtru 
 function clickThruFunc($mysqli, $username, $params){    
     $params = json_decode($params, true);
     $savedSearch = getSavedsearchClickThru($mysqli, $params) ?? "Last Search";
     $params[] = array("regular"=>array("searchSavesearch"=>$savedSearch)); 
          
     $params = json_encode($params, JSON_FORCE_OBJECT);
     return applySmartSearch($mysqli, $username, $params);   
 }
 
 function getSavedsearchClickThru($mysqli, $params){
     require_once 'analyticsUtils.php';
     global $username;
     $search_type = key($params[0]);
     $data_arr = getDataFromParams($search_type, $params);
     $analytics_obj = new AnalyticsClass($mysqli, $data_arr['table'], null, $username, 0);
     $saved_search_arr = $analytics_obj->getSavedSearchName(false);
     $name = $mysqli->real_escape_string($saved_search_arr['name']);
     $sql = "SELECT NAME,max(date) FROM savedsearch WHERE NAME LIKE ('%$name%') AND username='$username'";
     $results = sqlQuery($mysqli,$sql);
     while ($obj = $results->fetch_object()){
         return $obj->NAME;
     }
     return NULL;
 }
 
 function getDataFromParams($type, $params){
     if($type=="regular"){
         $table = key($params[0][$type]);
         $tableData = $params[0][$type][$table];
         if(strpos($tableData, '=')!==false){//means - concept
             $tableArr = explode('=',$tableData);
             $table = $tableArr[0];
             $tableData = $tableArr[1];
        }
     }
     else{
         $tableData =   key($params[0][$type]);
         $table = key($params[0][$type][$tableData]);
     }
     
     return array('table'=>$table, 'tableData'=>$tableData);
     
 }
 
 function getSolrFields($mysqli, $username, $isAdmin){
     global $solr_skip_fields;
     $useSolr = getSystemSettingsProp($mysqli, "useSolr");
     
     if(!$useSolr)
     {
         return;
     }
     
     global $SOLR_SERVER_HOSTNAME, $SOLR_SERVER_PORT, $SOLR_COLLECTION, $SOLR_TXT_FIELD;
     $solrQuery = "http://$SOLR_SERVER_HOSTNAME:$SOLR_SERVER_PORT/solr/$SOLR_COLLECTION/select?wt=csv";
     if (!$isAdmin){
         $solrQuery .= "&fq=username:$username";
     }
     $solrString = file_get_contents($solrQuery);
     write_to_log("TRACE", "solrQuery:$solrQuery");
     $solrResponse = explode(',',$solrString);
     $solrResponse[count($solrResponse)-1] = preg_replace('/\n$/','',$solrResponse[count($solrResponse)-1]);
     foreach($solrResponse as $item){
         if(!in_array($item, $solr_skip_fields)){
             $res[] = $item;
         }
     }
     return $res;
 }
 
 
 //fetch the accesslogs from TRACE-LOG
 function getLogs($action){
     global $TEMP_FOLDER;
     
     write_to_log("TRACE", "Getting logs for 'login'");
     
     header("Content-type: text/plain");
     
     
     $files = glob($TEMP_FOLDER.'/TRACE_log-2*');
     foreach($files as $log_file){
         $handle = fopen($log_file, "r");
         if ($handle) {
             while (($line = fgets($handle)) !== false) {
                 if(strpos($line, $action)!==false){
                     echo $line;
                }
             }
             
             fclose($handle);
         } else {
             write_to_log("TRACE", "Error open log file");
             
         } 
          
     }
 }
