<?php
use PHPMailer\PHPMailer\PHPMailer;
use PHPMailer\PHPMailer\SMTP;
use PHPMailer\PHPMailer\Exception;

class BackgroundFunctions {
    
    private $aggregated_sentiment = array();
    private $alertsArray = [];
    private $annotElementsParentClasses = array();
    private $annotElementsSubclassesKey = array();
    private $arrGeneral = array();
    private $arr_d = array();
    private $arr_word = array();
    private $checkIfExistsInDigest_hash = array();
    private $class2subClasses = array();
    private $concept2name = array();
    private $conceptsSentiment = array();
    private $dcid = null;
    private $defaultLanguage = "";
    private $digestJson = array();
    private $doc_username = [];
    private $emails_sql = array();
    private $emotion_arr = array();
    private $emotion_concept = 0;
    private $entities_alert_wl;
    private $extraFieldsObject = array();
    private $fileFields = array();
    private $ivEntityMatcherService;
    private $key2idConcept = array();
    private $manageEntitiesArr = array();
    private $hiddenEntitiesArr = array();
    private $merged_data = array();
    private $metaParameter = array();
    private $meta_arr;
    private $mysqli = null;
    private $tables = array();
    private $newText = "";
    private $orgTextKeyArr = array();
    private $orgTextValueArr = array();
    private $org_keys = array();
    private $personKey2idName = array();
    private $personStrippedName = [];
    private $persons_data = [];
    private $persons_names = [];
    private $pred2inst = array();
    private $pro_anti_array = array();
    private $qname2node = array();
    private $rules_json = [];
    private $sentimentArr = array();
    private $slotdata = array();
    private $solr_array = array();
    private $sqlTables = array();
    private $sql_array =  null;
    private $sql_final = [];
    private $sql_rules_final = [];
    private $subClass2classes = array();
    private $table2values = array();
    private $text = "";
    private $total_concepts = 0;
    private $updatedFileFields = array();
    private $useSolr = true;
    private $username = "";
    private $version = 1;
    private $wl_mysqli;
    private $xpathDom = null;
    
    public  function __construct($dcid){
        require_once 'parseUtils.php';
        require_once 'settings.php';
        global $version, $databaseName, $huge_files;
        $this->mysqli = sqlCreateConnection("backGround");
        
        $query =" show tables";
        $res = sqlQuery($this->mysqli, $query);
        $this->tables = $res->fetch_all ();
        if($huge_files){
            set_time_limit(0); //special flag for huge files as 200+mb pdf - need alot of time to run this script for 1 rdf
        }

       
        $databaseName_str = strtolower($databaseName);
        $this->wl_mysqli = sqlCreateDBConnection("wl_$databaseName_str");
        $this->sql_array = new InsertArray($this->mysqli);
        $this->pro_anti_array = new InsertArray($this->mysqli);
        $this->dcid = $dcid;   
        $this->getFileData();
        $this->username = $this->fileFields['username'];  
        $res = sqlQuery($this->mysqli,"SHOW TABLES");
        while ($row = $res->fetch_row()){
            $this->sqlTables[] = strtolower($row[0]);
        }
        $settings_class =  new Settings($this->mysqli, $this->username);
        $allConceptArrDB = $settings_class->getAllConceptsArr();
        $this->useSolr = getSystemSettingsProp($this->mysqli, "useSolr");    
        $concept_class = new AllConceptsData($this->mysqli, $allConceptArrDB, null,$this->username);
        $this->merged_data = array_change_key_case($concept_class->getMergedData()); 
        $this->manageEntitiesArr = $this->getManageEntities();      
        $this->hiddenEntitiesArr = $this->getHiddenEntities();
        $this->rules_json =  init_rules_wl();
        $this->ivEntityMatcherService = getSystemSettingsProp($this->mysqli, "ivEntityMatcherService");
        $this->entities_alert_wl = getAlertSettingsProp($this->mysqli, "entities_alert_list"); 
    }
    
    public function __destruct() {
        sqlClose($this->mysqli, 'backGround');
        print "\n $this->dcid Background: finished\n";
    }
    
    /**
     * @param string $b64
     * @desc set the text from a base64 argument
     */
    public function setText($b64)
    {
        $this->text = iconv("UTF-16LE", "UTF-8", base64_decode($b64));
        if($_REQUEST['getmetadata']){
            $this->meta_arr["intuview_data"]["full_text"] = $this->text;
        }
    }

    /**
     * @return $username
     */
    public function getMysqli()
    {
        return $this->mysqli;
    }
    
    /**
     * @return $merged_data
     */
    public function getMerged_data()
    {
        return $this->merged_data;
    }

    /**
     * @return $metaParameter
     */
    public function getMetaParameter()
    {
        return $this->metaParameter;
    }

    /**
     * @return $username
     */
    public function getUsername()
    {
        return $this->username;
    }
    
    /**
     * @return $text
     */
    public function getText()
    {
        return $this->text;
    }

    public function setxPathDom($element){
        $this->xpathDom = $element;
    }
    
    /**
     * getFileData()
     */
    public function getFileData(){
        write_to_log("TRACE", $this->dcid . " Background: fileFields");
        //check user access
        $sql = "SELECT * FROM files where id=$this->dcid";
        $res = sqlQuery($this->mysqli, $sql);
		$curr_fields = null;
		if(FALSE === $res) {
            write_to_log("ERROR", "background [id=$this->dcid] select from files failed ");
        } else {
			$curr_fields = $res->fetch_assoc();
			isset($curr_fields) || write_to_log("ERROR", "background [id=$this->dcid] unable to fetch file from DB " . print_r($curr_fields, true));
		} 				
		
		write_to_log("TRACE", $this->dcid . " Background: fileFields - DONE");
        $this->fileFields =  $curr_fields;
    }
    
    /**
     * getBlobData
     * @param  $this->mysqli
     * @return void|string
     */
    public function getBlobData(){
        global $xml_results_path;
        write_to_log("TRACE", $this->dcid . " Background: blob");
        
        $blob = file_get_contents($xml_results_path .DIRECTORY_SEPARATOR ."$this->dcid" . "_res.xml");
        if ($blob) {
            dumpDocsDbg("blob", $this->dcid, $blob, false);
        }
        write_to_log("TRACE", $this->dcid . " Background: blob DONE - strlen of blob" . strlen($blob));
        
        return $blob;
    }
    
    public function  handleExtraData(){
        //insert doc extra fields to DB
        self::setExtraFieldsObject();
        if (!empty($this->extraFieldsObject))
        {
            self::extraFieldObjectToDB($this->extraFieldsObject );
            //determine default language by extra data language (typically twitter language)
            $lang =  $this->extraFieldsObject ["lang"] ?? NULL;
            $this->defaultLanguage = lang_mapper($lang);
        }
    }
    
    public function setExtraFieldsObject() {
        global $extraColumns, $uploadedFilesPath, $extraDataKey2Table, $extraFiledsInsertionFunctions;
        $realUploadedFilesPath = realpath($uploadedFilesPath);
        if (empty($extraColumns) || empty($realUploadedFilesPath)){
            return;
        }
     
        $res = sqlQuery($this->mysqli, "SELECT filename, shortFilename, origFolder FROM files WHERE id = $this->dcid");
        $row = $res ? $res->fetch_object() : null;
        if (!$row) {
            write_to_log("ERROR", "setExtraFieldsObject failed");
            return;
        }
        $fileAndPath = docIdToFilePath($this->dcid, $this->mysqli, true);
  
        $realFolder = realpath($fileAndPath["filePath"]) . DIRECTORY_SEPARATOR;
        $filePath = $realFolder . $fileAndPath["fileName"];
        $extraFieldsFile = $filePath;//str_replace($realUploadedFilesPath, $realExtraFiledsPath, $filePath);

        $handle = NULL;
		$extraFieldsObject = array();
        $ext = pathinfo($extraFieldsFile, PATHINFO_EXTENSION);
        if (file_exists($extraFieldsFile) && strtolower($ext)=='json')
        {
            $filecontent = file_get_contents($extraFieldsFile);
         
            if (isJson($filecontent)) {
                $extraFieldsObject  = json_decode($filecontent, true);
                if($_REQUEST["getmetadata"]){
                    $this->meta_arr["intuview_data"]["org_json"] = $extraFieldsObject;
                }
            } else {
                write_to_log("ERROR", "File $extraFieldsFile is in invalid format ");
            }
        } else {
            if (!file_exists($extraFieldsFile)) {
                $extraFieldsFile1 = str_replace(".txt", "_.txt", $extraFieldsFile);
                if (file_exists($extraFieldsFile1)) {
                    $extraFieldsFile = $extraFieldsFile1;
                } else if (array_key_exists("shortFilename", $fileAndPath)) {
                    //TODO: fix this, as shortFilename of the file won't match extra fields file
                    $extraFieldsFile1 = copyFileToTemp($realFolder . $fileAndPath["shortFilename"], $this->dcid, "_extra");
                    if (file_exists($extraFieldsFile1)) {
                        $extraFieldsFile = $extraFieldsFile1;
                    }
                }
            }
            if (!file_exists($extraFieldsFile)) {
                write_to_log("TRACE", "extra fields file doesn't exist: $extraFieldsFile");
                $this->extraFieldsObject["filename"] = $fileAndPath["fileName"]; //add the file name to solr
                
                return;
            }
            $handle = fopen($extraFieldsFile, 'r');
            while (($line = fgets($handle)) !== FALSE) {
                if (strpos($line, UTF8_BOM) === 0) {
                    $line = str_replace(UTF8_BOM, "", $line);
                }
                
                $line = preg_replace("/[\r\n]+$/u", "", $line); //miltibyte trim
                $keyVal = explode("=>", $line ?? "");
                if (count($keyVal) != 2) {
                    continue;
                }
                
                $key = $keyVal[0];
                $value = $keyVal[1];
                $extraFieldsObject [$key] = $value;
            }
        }
        if ($handle !== NULL) {
            fclose($handle);
        }
		$this->extraFieldsObject = array_change_key_case($extraFieldsObject);
		
		$this->extraFieldsObject["filename"] = $fileAndPath["fileName"]; //add the file name to solr
    }
    
    public function  extraFieldObjectToDB()
    {
        global $meta_to_ont_keys;
        global $isLangOverride, $extraColumns, $extraDataKey2Table, 
        $extraFiledsInsertionFunctions, $timestamp_fields;
        $extraDataKey2Table = array_change_key_case($extraDataKey2Table);
                
        foreach ($this->extraFieldsObject  as $key => $value2){
            //#TODO: cause delays - need to check
//             if(array_key_exists($key, $meta_to_ont_keys)){
//                 $this->addMetaToOnt($meta_to_ont_keys, $key, $value2); 
//             }
           
            $key = trim($key);
            //skip multi arrays values
            if(is_array($value2) && is_array(current($value2))){
                continue;
            }
            
            
            foreach (is_array($value2) ? array_unique($value2) : [$value2] as $value_org)
            {
                $value = toDbString($this->mysqli,$value_org, 254); //longer values
                if ($value==="0"|| empty($value)){
                    continue;
                }
                
                //add user_screen_name to authour
                if($key == "user_screen_name"){
                    $keys = "docId,value";//"docId,value, nm_info";
                    $values = "'$this->dcid','$value'";//"'$this->dcid','$value', '$person_data_str'";
                    
                    //check if name is valid - so add to WL and metadata
                    //if ($this->isNameValid($value)) {
                    $person_data = $this->addNameToWL($value_org, -1);
                    $person_data_arr = $this->filterNAData($person_data["Name Alternatives"]);
                    $person_data_str = json_encode($person_data_arr, JSON_UNESCAPED_UNICODE);
                        
                    //}
                    if( $_REQUEST['getmetadata']){
                        $this->meta_arr['intuview_data']['metadata']["author"] = ["author"=>$value, "info"=>$person_data_arr ?? "no info"];
                    }
                    $this->sql_array->add_element("file_author", $keys, $values);
                    
                }
      
                $tableName = NULL;
                //extraColumns contains table names as keys and captions as values
                foreach ($extraColumns as $k => $v)
                {
                    if (strtolower($key) != strtolower($v) && strtolower($key) != strtolower($k))
                        continue;
                        $tableName = (strtolower($key) == strtolower($k)) ? $k : $v;
                        break;
                }
                //map between different extra data name conventions (twitter vs. gnip) to our tables
                if (is_array($extraDataKey2Table) && array_key_exists(strtolower($key), $extraDataKey2Table)){
                    $tableName = $extraDataKey2Table[$key];
                }
                    
                
                if (empty($tableName))
                {
                    write_to_log("TRACE", "$key not found in extra columns");
                    continue;
                }
                if(in_array($tableName, $timestamp_fields)){
                    $value = date('m/d/Y H:i:s', $value);
                }
                
                //for alerts 
                if($key == "user_screen_name"){
                    $this->doc_username["name"] = $value;
                }
                if($key == "user_id_str"){
                    $this->doc_username["id"] = $value;
                }
                //End - for alerts 
                $value = toDbString($this->mysqli, $value);
                sqlQuery($this->mysqli, "INSERT INTO $tableName (docId,value)
                    VALUES (\"$this->dcid\",\"$value\")");
                
                //if needs special insertions (e.g. for other tables, like t_post_interactions)
                if (array_key_exists($tableName, $extraFiledsInsertionFunctions))
                {
                    $insertionFunction = $extraFiledsInsertionFunctions[$tableName];
                    $insertionFunction($this->mysqli, $tableName, $this->dcid, $value);
                }
            }
        }
        //if ($handle)
        //fclose($handle);
    }

    function filterNAData($na_arr){
        $ret = [];
        foreach($na_arr as $index=>$na){
            foreach($na as $key=>$data){   
                $key = $key == 'caption' ? "name" : $key;
                //add also ethnicity obj
                if($key == 'ethnicity'){
                    $ret[$index]['ethnicity_obj'] = $this->parseEthnicity2Obj($data);                  
                }
                $key = str_replace('_', ' ', $key);
                $key = ucwords($key);
                if(is_array($data) && array_key_exists("caption", $data)){
                    $ret[$index][$key] = $data["caption"];
                }
                else{
                    
                    $ret[$index][$key] = $data;
                }
            }
        }
        return $ret;
    }
    
    function isNameValid($name) {
        global $ignore_single_nc;
        // Define the allowed characters for a name
//         $allowedCharacters = '/^[\p{L}\s\'\\\\-]+$/u';
//         $allowedCharacters = '/^[\p{L}\s\'-]+$/u';
        
//         $res = preg_match($allowedCharacters, trim($name));
        $res = true;
        if($ignore_single_nc){           
            $arr = preg_split('/[\s-]+/', $name, -1, PREG_SPLIT_NO_EMPTY);
            $is_valid_count = count($arr) > 1;
            $res = $is_valid_count && $res; 
        }
        // Check if the name contains only allowed characters
        return $res;
    }
    /**
     * 
     * @param unknown $str
     */
    function parseEthnicity2Obj($string){
        $pattern = '/([^()]+)\s*\(([^()]+)\)|([^()]+)/';
        
        preg_match_all($pattern, $string, $matches, PREG_SET_ORDER);
        
        $result = [];
        
        foreach ($matches as $match) {
            if ($match[1] != '') {
                $result[trim($match[1])] =  intval(str_replace('%', '',$match[2]));
            } elseif ($match[3] != '') {
                $result[trim($match[3])] = '';
            }
        }
        asort($result);
        return $result;
    }
    
    /**
     * addUserLocationOnt - analyse the user-location - fetch instances and insert to user_location_ont table 
     * @param unknown $value
     */
    function addMetaToOnt($meta_to_ont_keys, $org_table, $value){
        $dest_table = $meta_to_ont_keys[$org_table]['dest_table'];
        $dest_type = $meta_to_ont_keys[$org_table]['type'];
        $dest_id = $this->getConceptId($dest_table);        
        if(is_array($value)){
            $instances = [];
            foreach($value as $v){
                $instaces_temp = getKbInstanceSlots($v,null, false, true);
                $instances = array_merge($instances, $instaces_temp);
            }
            
        }
        else{
            $instances = getKbInstanceSlots($value,null, false, true);   
        }
        
        foreach($instances as $inst){
            if(!array_key_exists("iv:entity-type", $inst)){
                continue;
            }
            $type = str_replace("http://www.intuview.com/ontology#", "",$inst["iv:entity-type"]);
            if($type == $dest_type){
                $ont_name = str_replace("http://www.intuview.com/ontology#", "", $inst["rdf:about"]);
                $caption = $inst['iv:caption']['en'];
                $table_columns = ['concept_type', 'concept_type_source', 'docId','name_version','ontName','ontUrl','username','occurrences','type','orgText_version'];
                $table_values = [$dest_id, $dest_id, $this->dcid,"\"$caption\"","\"$ont_name\"","\"$ont_name\"","\"$this->username\"","\"1\"","\"$type\"","\"$caption\""];
                $table_columns = "(".implode(',', $table_columns).")";
                $table_values = "(".implode(',', $table_values).")";
                $this->table2values[$dest_table][$table_columns]["values"][] = $table_values;
            }
        }
    }
    
    /**
     * getUseSolr 
     * @return boolean|NULL
     */
    function getUseSolr(){
        return $this->useSolr;
    }
    
    /**
     * addDataToSolr
     */
    function addDataToSolr($mysqli){
        //get searchKeywords alert settings
        $searchKeywords_str = getAlertSettingsProp($this->mysqli, 'searchKeywords');
        
        global $SolRoptions, $SOLR_COLLECTION, $dropdownArrSolr;
//         $response = @simplexml_load_file("http://$SolRoptions[hostname]:$SolRoptions[port]/solr/admin/cores?action=status&core=$SOLR_COLLECTION");
        $response = file_get_contents("http://$SolRoptions[hostname]:$SolRoptions[port]/solr/admin/cores?action=status&core=$SOLR_COLLECTION");
        
        if($response!==false){
            
            global $SOLR_COLLECTION, $SOLR_MAX_COMMIT_DELAY, $SOLR_TXT_FIELD;
            $solrQuery = "http://$SolRoptions[hostname]:$SolRoptions[port]/solr/$SOLR_COLLECTION/update/json";
            if ($SOLR_MAX_COMMIT_DELAY && empty($searchKeywords_str)){  //if we have alert terms - dont delay the commit, do it now
                $solrQuery .= "?commitWithin=$SOLR_MAX_COMMIT_DELAY";
            }
            else{
                $solrQuery .= "?commit=true";
            }
        $insert_array = array(
                "id" => $this->dcid,
                "$SOLR_TXT_FIELD" => $this->text,
                "username" => $this->username
            );
           

//         foreach ($dropdownArrSolr as $key){
//             $val = $this->extraFieldsObject[$key] ?? null;
//               if(!empty($val)){
//                   if(is_array($val)){
//                       foreach($val as $item){
//                           $insert_array += array($key=>$val);
//                       }
//                   }
//                   $insert_array += array($key=>$val);
//           }
//         }
        
//         foreach($this->solr_array as $table=>$elements){
//             foreach($elements as $elements_list){
//                 foreach($elements_list as $item){
//                     $insert_array[$table][] = $item;
                    
//                     $insert_array[$table.'_name'][] = $item['name'];
//                     $insert_array[$table.'_orgText'][] = $item['orgText'];
//                 }
//             }    
//         }

        //adding date to solr
        
        foreach($this->solr_array as $table=>$elements){
            $insert_array[$table][] = $elements;
        }
        
        $updateResponse = file_get_contents($solrQuery, false,
            stream_context_create(array("http" => array(
                "header" => "Content-type:application/json",
                "method" => "POST",
                "content" => json_encode(array($insert_array))
            ))));
        
        $solrResponse = json_decode($updateResponse);
        
        if (!$solrResponse || !$solrResponse->responseHeader || $solrResponse->responseHeader->status !== 0){
            write_to_log("ERROR", "Unexpected SolR update response");
        }
        else{
            write_to_log("TRACE", "Manage to add to SOLR");
        }
    }
    else{
        write_to_log("ERROR", "Solr is set on but cant be connected");
    }  
    
        
    if(!empty(trim($searchKeywords_str))){
        $searchKeywords = explode(" OR ", $searchKeywords_str);
        foreach($searchKeywords as $keyword){
            $keyword = trim($keyword, "'");
            $keyword_url =  urlencode($keyword);
            $solrQuery = "http://$SolRoptions[hostname]:$SolRoptions[port]/solr/$SOLR_COLLECTION/select?wt=json&indent=true&fl=id&start=0&rows=500&q=content_txt:$keyword_url+AND+ID=$this->dcid";
            
            $solrJson = file_get_contents($solrQuery);
            write_to_log("TRACE", "solrQuery:$solrQuery");
            $solrResponse = json_decode($solrJson);
            write_to_log("TRACE", "solrResponse:$solrJson");
            if ($solrResponse && $solrResponse->response && $solrResponse->response->docs){
                $this->alertsArray['keywords'][] = $keyword;
            }
        }
        
    }
}
    
    /**
     * insert_to_text_db
     * @param  $textDataBase
     */
    public function insert_to_text_db($textDataBase) {
        $mysqli_t = sqlCreateDBConnection($textDataBase);
        $t_m =  $mysqli_t->real_escape_string($this->text);
        $res = sqlQuery($mysqli_t, "INSERT INTO documents (id, TEXT, username) VALUES ($this->dcid, \"$t_m\", \"$this->username\")");
        if ($res === FALSE){
            write_to_log("WARNING", "background unable to insert document text id = $this->dcid  sql - " . $mysqli_t->error);
            $res = sqlQuery($mysqli_t, "UPDATE documents SET TEXT=\"$t_m\", username=\"$this->username\" WHERE id=$this->dcid");
            if ($res === FALSE) {
                write_to_log("ERROR", "background unable to update document text id = $this->dcid  sql - " . $mysqli_t->error);
            }
        }
        sqlClose($mysqli_t, 'background - insert_to_text_db', true);
    }
    
    public function addDocToSolr(){
        global $SolRoptions, $SOLR_COLLECTION;
//         $response = @simplexml_load_file("http://$SolRoptions[hostname]:$SolRoptions[port]/solr/admin/cores?action=status&core=$SOLR_COLLECTION");
        $response = file_get_contents("http://$SolRoptions[hostname]:$SolRoptions[port]/solr/admin/cores?action=status&core=$SOLR_COLLECTION");
        
        if($response!==false){
            
            global $SOLR_COLLECTION, $SOLR_MAX_COMMIT_DELAY, $SOLR_TXT_FIELD;
            $solrQuery = "http://$SolRoptions[hostname]:$SolRoptions[port]/solr/$SOLR_COLLECTION/update/json";
            if ($SOLR_MAX_COMMIT_DELAY){
                $solrQuery .= "?commitWithin=$SOLR_MAX_COMMIT_DELAY";
            }
            else{
                $solrQuery .= "?commit=true";
            }
            $updateResponse = file_get_contents($solrQuery, false,
                stream_context_create(array("http" => array(
                    "header" => "Content-type:application/json",
                    "method" => "POST",
                    "content" => json_encode(array(array(
                        "id" => $this->dcid,
                        "$SOLR_TXT_FIELD" => $this->text,
                        "username" => $this->username
                    )))
                ))));
            $solrResponse = json_decode($updateResponse);
            if (!$solrResponse || !$solrResponse->responseHeader || $solrResponse->responseHeader->status !== 0){
                write_to_log("ERROR", "Unexpected SolR update response: " . $updateResponse);
            }
        }
        else{
            write_to_log("TRACE", "Solr is set on but cant be connected");
        }
    }
    
    
    public function insertRdfResults($tripples){
        $aggr = $tripples->getElementsByTagName("AggregatedRDF")->item(0);
        $content = $aggr->firstChild;
        if ($content){
            rdf2structs("agg", $content->wholeText, $this->qname2node, $this->pred2inst, $this->class2subClasses, $this->subClass2classes);

        }
        try{
            $digest = $tripples->getElementsByTagName("DigestRDF")->item(0);
            $content = $digest->firstChild;
            
            if ($content){
                rdf2structs("dig", $content->wholeText, $this->qname2node, $this->pred2inst, $this->class2subClasses, $this->subClass2classes);

            }
            
            TableTree::createTreeTables($this->mysqli, false);
            TableTree::insertParentTables($this->mysqli, $this->subClass2classes);
            
            TableTree::insertSubTables($this->mysqli, $this->class2subClasses);
            
            //add KB RDF - if exists
            $kbRDF = $tripples->getElementsByTagName("KbRDF");
            
            if($kbRDF){
                $kbRDF = $kbRDF->item(0);
            }
            if($kbRDF){
                $kbRDF = $kbRDF->firstChild;
            }
            
            if ($kbRDF)
            {
                dumpDocsDbg("qname2node1", $this->dcid, $this->dcid, $this->qname2node);
                rdf2structs("dig", $kbRDF->wholeText, $this->qname2node, $this->pred2inst, $this->class2subClasses, $this->subClass2classes);

            }

            
            dumpDocsDbg("qname2node", $this->dcid, $this->qname2node);
            dumpDocsDbg("pred2inst", $this->dcid, $this->pred2inst);
            
        }
        catch (Exception $e)
        {
            write_to_log("WARNING", $this->dcid . " exception");
        }
        
    }
    
    
    public function metaParamsHandler(){
        $docModel = $this->pred2inst["dig"]["rdf:type"]["http://www.intuview.com/ontology#Document-model"][0];
        if ($docModel){
            foreach ($docModel as $key => $value)
            {
                //skip uninitialized dates
                if (strpos($key, "-date") && $value == '0000-00-00 00:00:00'){
                    continue;
                }
                
                $key = str_replace("-", "_", str_replace("iv:", "", unqualifyName($key)));
                
                if (is_array($value) && array_key_exists("en", $value)){
                    $value = $value["en"];
                }
                
                $arr = is_array($value)? $value : array($value);
                foreach ($arr as $k => $v)
                {
                    if (array_key_exists($v, $this->qname2node["dig"]) && array_key_exists("iv:caption", $this->qname2node["dig"][$v])){
                        $v = $this->qname2node["dig"][$v]["iv:caption"];
                    }
                    if (is_array($v)){
                        $v = $v["en"] ?? null;
                    }
                    
                    $val = ucfirst(str_replace("language-","", unqualifyName($v)));
                    //in case domain caption is not passed by intuScan - fake caption...
                    $val = str_replace("-domain-parameter", "", $val);
                    //convert dates from format dd/mm/yyyy to yyyy-mm-dd
                    if (strpos($key, "date") !== FALSE)
                    {
                        $ymd = array_reverse(explode("/", $v));
                        if (count($ymd) != 3){
                            continue; //not a date
                        }
                        //normalize unknown day (00) to first day of the month (01)
                        if ($ymd[2] == "00"){
                            $ymd[2] = "01";
                        }
                        //normalize unknown year (0000) to current year
                        if ($ymd[0] == "0000"){
                            $ymd[0] = date('Y');
                        }
                        $val = join("-", $ymd);
                    }
                    if (array_key_exists($key, $this->metaParameter)){
                        $this->metaParameter[$key] .= "; ".$val;	//append multiple value
                    }
                    else{
                        $this->metaParameter[$key] = $val;
                    }
                }
            }
            dumpDocsDbg("mp", $this->dcid, $this->metaParameter);
        }
    }
    
    /**
     * updateUserStatus - update user_status table about file size
     */
    public function updateUserStatus(){
        //update user_status table about file size
        $query = "SELECT docNumber,fileSizeSum FROM user_status WHERE username=\"$this->username\"";
        $res = sqlQuery($this->mysqli,$query);
        if($res != TRUE){
            $row = $res->fetch_object();
            
            $d = $row->docNumber;
            $s = $row->fileSizeSum;
            
            $d += 1;
            $s += ($this->metaParameter[file_size])/1000;
            $sql = "UPDATE user_status
    				  SET docNumber=$d,fileSizeSum=$s
    				  WHERE username=\"$this->username\"";
            sqlQuery($this->mysqli,$sql);
        }
    }
    
    
    public function handleDateField(){
        //update date_field table if not exsits
        $date_extradata = $this->update_date_field_extraData();
        if(empty($date_extradata)){
            global $isSocialDateFormat;
            $date_field_val = $this->update_date_field();
            //if the value is set - we will override the metaparameter['date'] value (would be from intuscan so no need for this date - just incorrect)
            if(!empty($date_field_val) && ($isSocialDateFormat || empty($this->metaParameter['date']))){
                $this->metaParameter['date'] =  $date_field_val;
            }
        }
        else{
            $this->metaParameter['date'] =  $date_extradata;
        }
        
        $this->solr_array["date"] = $this->metaParameter['date']; //save in solr date
    }
    
    public function insertMetaParameters(){
        global $dropdownArr;
        $domainArr = array();
        $topicArr = array();
        foreach ($this->metaParameter as $k => $v){
            if ($k == "emotion")
            {
                continue;
            }
            if ($k == "threat_rating")
            {
                $temp = explode("-risk-parameter", $v);
                $v = $temp[0];
            }
            if ($k == "domain"){
                $e = explode("; ", $v);
                foreach ($e as $t){
                    $domainArr[$t] = $t;
                    $keys = "docId,value";
                    $t = toDbString($this->mysqli, $t);
                    $values = "'$this->dcid','$t'";
                    $this->sql_array->add_element("domain", $keys, $values);
                }
            }
            if ($k == "document_topic"){
                $topicMapping = array(
                    "Ied" => "Explosives",
                    "IT" => "Technology",
                    "Economy-finance" => "Finance",
                    "Explosive-ied" => "Explosives",
                    "Islam" => "Islamic",
                    "Medical" => "Health",
                    "Physical-science" => "Physical-sciences",
                    "Sport" => "Sports",
                    "Technology-it" => "Technology"
                );
                $e = explode("; ", $v);
                foreach ($e as $t){
                    $temp = explode("-topic-concept", $t);
                    $tpc = $temp[0];
                    if (array_key_exists($tpc, $topicMapping)){
                        $t = $topicMapping[$tpc];
                    }
                    if ($t == "Islam"){
                        $t = "Islamic";
                    }
                    
                    $topicArr[$t] = $t;
                    $t = trim(toDbString($this->mysqli, $t));
                    
                    $keys = "docId,value";
                    
                    $values = "'$this->dcid','$t'";
                
                    $this->sql_array->add_element("topic", $keys, $values);
                 
                }
            }
            
            if ($k == "author"){
                $k = "file_author";
            }
            if ($k == "recipient"){
                $k = "to_field";
            }
        
            if (array_key_exists($k,$dropdownArr) || $k == 'file_author'){
                $e = explode("; ", $v);
                $langOverRide = $this->checkLangOverRide($k);
                
                foreach ($e as $t){
                    if($k == "file_author"){         
//                         $keys = "docId,value, nm_info";+
                        $keys = "docId,value";
                        $value = toDbString($this->mysqli, $t);
                        $person_data = $this->addNameToWL($value, -1);
//                         $person_data_str = json_encode($this->filterNAData($person_data["Name Alternatives"]), JSON_UNESCAPED_UNICODE);
//                         $values = "'$this->dcid','$value', '$person_data_str'";
                        $values = "'$this->dcid','$value'";
                        $this->sql_array->add_element("file_author", $keys, $values);     
                                              
                    }
                    elseif ($k != "document_topic" && $k != "domain"){
                        $t = $langOverRide ?? $t;
                        $t = trim(toDbString($this->mysqli, $t));
                        $keys = "docId,value";
                        $values = "'$this->dcid','$t'";
                        $this->sql_array->add_element($k, $keys, $values);
                        
                    }
                    
                    if( $_REQUEST['getmetadata']){
                        $this->meta_arr['intuview_data']['metadata'][$k] = $t;
                    }
                }
                
                
            }
            
            else if (array_key_exists($k, $this->fileFields)){
                if ($this->fileFields[$k] != $v)
                {
                    if(empty($v)){
                        continue;
                    }
                    if (empty($this->fileFields[$k]) || $k == "date"){
                        $this->updatedFileFields[$k] = $v;
                    }
                    
                    //if not other date - use scan_time as date
                    if ($k == "scan_time" && empty($this->fileFields["date"]) && !array_key_exists("date", $this->metaParameter)){
                        $this->updatedFileFields["date"] = $v;
                    }
                }
            }
            else
            {
                write_to_log("TRACE", "background of $this->dcid: Skipping meta-parameter which is not a column in 'files' table: $k");
            }
        }
        global $restrictedTopicArr;
        $v_arr = [];
        foreach ($domainArr as $v){
            if ($v != "Islamic" && !array_key_exists($v, $restrictedTopicArr)){

                $keys = "docId,value";
                $v = toDbString($this->mysqli, $v);
                $values = "'$this->dcid','$v'";
                $this->sql_array->add_element("document_topic", $keys, $values);   
            }
            
        }
        foreach ($topicArr as $v){
            if ($v != "Islamic" && !array_key_exists($v, $restrictedTopicArr)){
                $v = toDbString($this->mysqli, $v);
                $keys = "docId,value";
                $values = "'$this->dcid','$v'";
                $this->sql_array->add_element("document_topic", $keys, $values);
 
                
            }
        }
        
        if (array_key_exists("Islamic", $topicArr)){
                
            $keys = "docId,value";
            $values = "'$this->dcid','Islamic'";
            $this->sql_array->add_element("document_topic", $keys, $values);        
        }
        
        if (!empty($this->updatedFileFields))
        {
            $query = "UPDATE files SET";
            $isFirst = true;
            foreach ($this->updatedFileFields as $k => $v)
            {
                if (!$isFirst){
                    $query .= ",";
                }
                $isFirst = false;
                $escapedV = toDbString($this->mysqli, $v);
                $query .= " $k=\"$escapedV\"";
            }
            $query .= " WHERE id=$this->dcid";
            sqlQuery($this->mysqli, $query);
        }
    }
    
    public function getAnnotElements(){
        $final_array = array();
        foreach($this->merged_data as $element){
            $className = $element["name"];
            $digestCaption = $element["digestCaption"];
            $class_flavor = $element["flavor"];
          
            $annotElementsSubclasses[$className] =  $this->class2subClasses[$className] ?? null;
            if ($annotElementsSubclasses[$className]){
                foreach ($annotElementsSubclasses[$className] as $subclass)
                {
                 
                    //avoid override for specific "parent" class if its under -topic (other branch)
                    if(!empty($this->annotElementsParentClasses[$subclass]['className'])
                        && strpos($className, '-topic') !== false){
                        continue;
                    }
          
//                     $this->annotElementsParentClasses[$subclass]['className'] = $className;
//                     $this->annotElementsParentClasses[$subclass]['digestCaption'] = $digestCaption;
//                     $this->annotElementsParentClasses[$subclass]['flavor'] = $class_flavor;
                    
                    $this->annotElementsParentClasses[$subclass][] = ['className' => $className, 'digestCaption'=>$digestCaption, 'flavor'=>$class_flavor];
                    //probably obsolete
                    if (!isset($this->annotElementsSubclassesKey[$subclass]) || !array_key_exists($subclass, $this->annotElementsSubclassesKey)){
                        $this->annotElementsSubclassesKey[$subclass] = $className;
                    }
                }
            }
            $this->annotElementsParentClasses[$className][] = ['className' => $className, 'digestCaption'=>$digestCaption, 'flavor'=>$class_flavor];
            
//             $this->annotElementsParentClasses[$className]['className'] = $className;
//             $this->annotElementsParentClasses[$className]['digestCaption'] = $digestCaption;
//             $this->annotElementsParentClasses[$className]['flavor'] = $class_flavor;
            
            if (!array_key_exists($className, $this->annotElementsSubclassesKey)){
                $this->annotElementsSubclassesKey[$className] = $className;
            }
        }
        
        dumpDocsDbg("annotElementsSubclasses", $this->dcid, $annotElementsSubclasses);
        dumpDocsDbg("annotElementsSubclassesKey", $this->dcid, $this->annotElementsSubclassesKey);
        dumpDocsDbg("annotElementsParentClasses", $this->dcid, $this->annotElementsParentClasses);
        
        $final_array['annotElementsSubclasses'] = $annotElementsSubclasses;
        $final_array['annotElementsSubclassesKey'] = $this->annotElementsSubclassesKey;
        $final_array['annotElementsParentClasses'] = $this->annotElementsParentClasses;
        
        return $final_array;
        
        
    }
    
    public function createDigestJson(){
        //we don't add to Aggregated-Digest emails and phone numbers, but we do want to display them in digestJson
        global $nonAggregatedTreatedAsAggregated, $virtualConcepts;
        global  $excluded_concepts_sql, $possible_identification_replacement;
        if (!empty($virtualConcepts)){
            $nonAggregatedTreatedAsAggregated = array_merge($nonAggregatedTreatedAsAggregated, $virtualConcepts);
        }
        foreach($this->merged_data as $element){
            $possibe_array = []; // for possible identification elements
            
            $exception_xcap = false;
            $typeInstances = [];
            $typeName = qualifyName($element["name"], "ont");
            if(array_key_exists("agg", $this->pred2inst) && array_key_exists("iv:entity-type", $this->pred2inst["agg"]) && array_key_exists($typeName, $this->pred2inst["agg"]["iv:entity-type"])){
                $typeInstances = $this->pred2inst["agg"]["iv:entity-type"][$typeName];
            }
            else if (array_key_exists("agg", $this->pred2inst) && array_key_exists($typeName, $this->pred2inst["agg"]["rdf:type"])){
                $typeInstances = $this->pred2inst["agg"]["rdf:type"][$typeName];
            }
            //special case for emails/phone numbers...
            else if (in_array($element["name"], $nonAggregatedTreatedAsAggregated) && array_key_exists($typeName , $this->pred2inst["dig"]["rdf:type"])){
                $temp_typeInstances = $this->aggregateFromPredicates($this->pred2inst["dig"]["rdf:type"][$typeName]);
                foreach($temp_typeInstances as $key => $val) {
                    if($key === 0){
                        $typeInstances = $temp_typeInstances[$key];
                        $typeInstances["iv:aggregated-from"] = is_array($val["iv:aggregated-from"]) ? $val["iv:aggregated-from"] : [$val["iv:aggregated-from"]];           
                        continue;
                    }
                    if(is_array($val["iv:aggregated-from"])){
                        foreach($val["iv:aggregated-from"] as $agg_temp){
                            $typeInstances["iv:aggregated-from"][] = $agg_temp;
                        }
                    }
                    else{
                        $typeInstances["iv:aggregated-from"][] = $val["iv:aggregated-from"];
                    }
                }
                $typeInstances = $temp_typeInstances;//array($typeInstances);
                $exception_xcap = true;
            }
            else{
                continue;
            }
            $menuChildren = array();
            $menuChildren_org = array();
            if ($typeInstances){
                for ($i = 0; $i < count($typeInstances); $i++) {
                    $deleteInstance = false;
                    $tInst = $typeInstances[$i];
                    $typesArr = $tInst["rdf:type"];
					if(is_string($typesArr)){
						$typesArr = [$typesArr];
					}
                    foreach($typesArr as $t1){
                        $t1 = str_replace('http://www.intuview.com/ontology#', '', $t1);
                        $current_sub = $this->subClass2classes[$t1] ?? [];
                        array_push($current_sub, $t1);
                        if(!empty(array_intersect($excluded_concepts_sql, $current_sub))){
                            $deleteInstance = true;
                        }
                    }
                    if($deleteInstance){
                        $deleteInstance = false;
                        unset($typeInstances[$i]);
                        continue;
                    }
                    
                  
                    $entityType = $tInst["iv:entity-type"] ?? $tInst["rdf:type"] ?? "";
                    
                    $xqname = $tInst["rdf:about"];
                    $xkey = unqualifyName($xqname);
                    $aggFrom = $tInst["iv:aggregated-from"];
                    if (!$aggFrom){
                        continue;
                    }
          
                    $xcap = $this->guessCaptionFromIvname($tInst);//!$exception_xcap ?  $this->guessCaptionFromIvname($tInst) : $element["name"];

                    if (empty($xcap)){
                        continue;
                    }
                    
       
                    $type = str_replace("http://www.intuview.com/ontology#", "",$entityType);  
                    $table = TablesInformation::getTableName($type);
                    
                    if($type=='Person-object' && !$this->isNameValid($xcap)){
                        write_to_log("ERROR", "person name isn't valid - $xcap");
                        continue;
                    }

                    /**
                     * check hidden and manageEntites & crossEntities
                     */
                    //check hidden entities (ignore them)
                    $isHidden = $this->checkHiddrenEntities($table, $xcap);
                    if($isHidden){
                        continue;
                    }
                    //check in aggregated or cross entities 
                    $aggCeArr = $this->checkInAggOrCross($table, $xcap);
                    $xcap = $aggCeArr["name_agg"] ?? $xcap;
                    $type = $aggCeArr["destConcept"] ?? $type;
                    
                                   /**
                     * =====
                     */
                    $is_ic = array_key_exists("iv:transliteration-ic",$tInst);
                    
                    $ontUrl = array_key_exists("iv:instance",$tInst)?$tInst["iv:instance"]:"";
                    $ontUrl = str_replace('http://www.intuview.com/ontology#', '', $ontUrl);
                    
                    
                    $isNotSingleChild = is_array($aggFrom);
                    $inner_span_key = self::get_key_forNoIndex($aggFrom);
                    
                    //For same class only - convert Organizational-identity to use the concept in iv:possible-identification (for collective group only)
                    if($type == "Organizational-identity"){
                        if(is_string($aggFrom) && array_key_exists($aggFrom, $this->qname2node["dig"])){
                            $aggedInst = $this->qname2node["dig"][$aggFrom];
                            if(array_key_exists("iv:possible-identification", $aggedInst)){
                                
                                $possible_identification = $this->checkPossibleIdentification($aggedInst);
                                
                                //if found - skip insert to org and added it to possible_array
                                if(!empty($possible_identification)){
                                    $possibe_array[] = array('title' => $xcap, 'key' =>  $inner_span_key, 'ontUrl' => $ontUrl, 'type' => $possible_identification, 'entity-type' => $possible_identification); 
                                    continue;
                                }
                            }
                        }
                    }
                    
                    unset($childMenu);
                    unset($childMenu_org);
                    if($isNotSingleChild) {
                        $childMenu = array('title' => $xcap, 'key' => $inner_span_key,  'icon' => false, 'ontUrl' => $ontUrl, 'children' => array(), 'type' => $entityType, 'entity-type' => $entityType);
                        $menuChildren[] = &$childMenu;
                        $ichildMenu = &$childMenu['children'];
                        
                        $childMenu_org = array('title' => $xcap, 'key' => $xkey,  'ontUrl' => $ontUrl, 'children' => array(), 'type' => $entityType, 'entity-type' => $entityType);
                        $menuChildren_org[] = &$childMenu_org;
                        $ichildMenu_org = &$childMenu_org['children'];
                    }
                    else
                    {
                        $aggFrom = array($aggFrom);
                        $ichildMenu = &$menuChildren;
                        $ichildMenu_org = &$menuChildren_org;
                    }
                    $check_exists_arr = [];
                    
                    foreach ($aggFrom as $af)
                    {
                        
                        
                        $this->checkIfExistsInDigest($ontUrl);  //register instance to avoid duplication later in digest
                        
                        $ykey = unqualifyName($af);
                        $y = $this->qname2node["dig"][$af];
                        if (!$y){
                            continue;
                        }
                        $xcap =  $aggCeArr["name_agg"] ?? ($is_ic ? $xcap : $this->guessCaptionFromIvname($y)) ;
                        
                        if (!$xcap){
                            continue;
                        }

                        $elements = $this->getSpanElements($af);
                        if (is_null($elements)){
                            continue;
                        }
                        
                        $sentiments = [];
                        foreach (["anti", "pro"] as $sent)
                        {
                            if (!array_key_exists("iv:sentiment-".$sent, $y)){
                                continue;
                            }
                            
                            $sents = $y["iv:sentiment-".$sent];
                            if (!is_array($sents)){
                                $sents = [$sents];
                            }
                            
                            foreach ($sents as $sentInstQName)
                            {
                                $sentInstType = $this->getSentInstType($sentInstQName);
                                $sentInstCaption = isset($this->qname2node["dig"][$sentInstQName]["iv:caption"]) ? $this->qname2node["dig"][$sentInstQName]["iv:caption"] : "";
                                if (is_array($sentInstCaption)){
                                    $sentInstCaption = $sentInstCaption["en"];
                                }
                                $sentiments[$sent][] = ["instance" => $sentInstCaption, "concept" => $sentInstType];
                            }
                        }
                        foreach ($elements as $delement) {

                                $span_key = self::createSpanKey($delement->getAttribute("end"));
                                $checkup_key = $span_key."_".$ontUrl;
                                if(!array_key_exists($checkup_key, $check_exists_arr)){
                                    $check_exists_arr[$checkup_key] = 1;  
                                }
                                else{
                                    continue;
                                }
                                
                                $this->arr_d[$delement->getAttribute("start")] = array($delement->getAttribute("end"), $span_key, $element['flavor']);
                                
                                $newChild = array('title' => $xcap, 'key' =>  $span_key, 'ontUrl' => $ontUrl, 'type' => $entityType, 'entity-type' => $entityType);
                                $newChild_org = array('title' => $xcap, 'key' => $ykey, 'span_key' =>  $span_key,'ontUrl' => $ontUrl, 'type' => $entityType, 'entity-type' => $entityType);
                                
                                if (!empty($sentiments)){
                                    $newChild["sentiments"] = $sentiments;
                                }
                                
                                $ichildMenu[] = $newChild;
                                $ichildMenu_org[] = $newChild_org;
                              
                            }
                    }
                }

            }
            foreach(array($menuChildren,$menuChildren_org) as $menu_arr){
                //sort by caption
                usort($menu_arr, array($this,'compareTitle'));
                foreach ($menu_arr as $child){
                    if (array_key_exists('children', $child)){
                        usort($child['children'], array($this,'compareTitle'));
                    }
                }
            }
            
            $icon = "default.gif";
            if(!empty($menuChildren)){
                $this->digestJson[] = array('title' =>  $element["name"], 'digestCaption' =>  $element["digestCaption"],  'flavor' =>  $element["flavor"], 'key' => $element["name"], 'isFolder' => true, 'children' => $menuChildren, 'type' => $tInst["rdf:type"], 'entity-type' => $entityType);
                $this->org_keys[] = array('title' =>  $element["name"], 'key'=>$element["name"],  'children' => $menuChildren_org, 'type' => $entityType, 'entity-type' => $entityType); 
            }
            if(!empty($possibe_array)){
                $first_pos = $possibe_array[0];
                $newValue =  $first_pos["type"];
                $newDigestArr = array_fill(0, count($element["digestCaption"]), $newValue);
                
                $this->digestJson[] = array('title' =>  $first_pos["type"], 'digestCaption' =>  $newDigestArr,  'flavor' =>  $element["flavor"],
                    'key' => $element["type"], 'isFolder' => true, 'children' => $possibe_array, 'type' => $first_pos["type"], 'entity-type' => $element["type"]);
 
            }
        }
        dumpDocsDbg("digestJson", $this->dcid, $this->digestJson);
        dumpDocsDbg("org_keys", $this->dcid, $this->org_keys);
    }
    
    /**
     * checkPossibleIdentification - check if the org element have the slot possible-identification - and if we need to create new element for digest json
     * @param unknown $possibe_array
     * @param unknown $aggedInst
     * @return string|NULL
     */
    public function checkPossibleIdentification($aggedInst){
        global $possible_identification_replacement;
        $possible_identification = str_replace("http://www.intuview.com/ontology#", "",$aggedInst["iv:possible-identification"]);
        $possible_identification_arr = explode('dynamic-inst-rt-',$possible_identification);
        $dash_arr = explode('-',$possible_identification_arr[1]);
        $allExceptLast = array_slice($dash_arr, 0, -1);
        $possible_identification = implode("-",$allExceptLast);
        if(in_array($possible_identification, $possible_identification_replacement)){
            return $possible_identification;
        }
        return null;
    }
    
    public function createSpanKey($end_index){
        return "span_key_$end_index";
    }
    
    public function get_key_forNoIndex($aggFrom){
        $first_element =  is_array($aggFrom) ? $aggFrom[0] : $aggFrom;   
        $inner_span = $this->getSpanElements($first_element);
        if(empty($inner_span)){
            return null;
        }
        return self::createSpanKey($inner_span[0]->getAttribute("end"));
    }
    
    public function createArrGeneral(){
        global $excluded_concepts_sql;
        if ($this->annotElementsSubclassesKey)
        {
            $name2Instances = $this->pred2inst["dig"]["iv:name"];
            
            dumpDocsDbg("name2InstancesTmp", $this->dcid, $name2Instances);
            foreach ($name2Instances as $ivname => $instances)
            {
                foreach ($instances as $inst)
                {
                    if($this->checkIfExistsInDigest($inst["rdf:about"])){
                        continue;
                    }
                    $insttypearray = $inst["rdf:type"];
                   $instEtntityType = $this->getExistsTable($inst["iv:entity-type"] ?? $insttypearray);
//                     $instEtntityType = $inst["iv:entity-type"] ?? "";
                    
                    if (!is_array($insttypearray)) {
                        $insttypearray = [ $insttypearray ];
                    }
                    foreach ($insttypearray as $multype)
                    {
                        $t = unqualifyName($multype);
                        if (!key_exists($t, $this->annotElementsParentClasses)){
                            continue;
                        }
                        
                        $xcap = $this->guessCaptionFromIvname($inst);
                        if (!$xcap){
                            continue;
                        }
                        $x =  $inst["rdf:about"];
                        
                        //patch - hide Non-contemporary-person 
                        $current_sub = $this->subClass2classes[$t] ?? [];
                        array_push($current_sub, $t);
                        if(!empty(array_intersect($excluded_concepts_sql, $current_sub))){
						    continue;
                        }
						
                        $parentArray = $this->annotElementsParentClasses[$t];
                        foreach($parentArray as $parent_element){
                            $parentClass = $parent_element['className'];
                            $digestClass = $parent_element['digestCaption'];
                            $flavorClass = $parent_element['flavor'];
                            $isNew = true;
                            if (isset($this->arrGeneral) && array_key_exists($parentClass, $this->arrGeneral)){
                                foreach ($this->arrGeneral[$parentClass] as $instance){
                                    if ($instance["x"]["value"] == $x){
                                        $isNew = false;
                                    }
                                }
                            }
                            
                            /**
                             * check hidden and manageEntites & crossEntities
                             */
                            //check hidden entities (ignore them)
                            $type = str_replace("http://www.intuview.com/ontology#", "",$instEtntityType ?? ""); 
                            if(!empty($type)){
                                $table = TablesInformation::getTableName($type);
                                
                                $isHidden = $this->checkHiddrenEntities($table, $xcap);
                                if($isHidden){
                                    continue;
                                }
                                if($type=='Person-object' && !$this->isNameValid($xcap)){
                                    write_to_log("ERROR", "person name isn't valid - $xcap");
                                    continue;
                                }
                                //check in aggregated or cross entities
                                $aggCeArr = $this->checkInAggOrCross($table, $xcap);
                                $xcap = $aggCeArr["name_agg"] ?? $xcap;
                                $type = $aggCeArr["destConcept"] ?? $type;
                            }
                            
     
                            //For same class only - convert Organizational-identity to use the concept in iv:possible-identification
//                             if($parentClass == $type && array_key_exists("iv:possible-identification", $inst) && $type == "Organizational-identity"){
//                                // $possible_identification = $this->checkPossibleIdentification($inst);
                                
//                                 $possible_identification = str_replace("http://www.intuview.com/ontology#", "",$inst["iv:possible-identification"]);
//                                 if(in_array($possible_identification, $possible_identification_replacement)){
//                                     $type = str_replace("http://www.intuview.com/ontology#", "", $possible_identification);
//                                     $parentClass = $type;
//                                     $digestClass = array_replace($digestClass, array_fill_keys(array_keys($digestClass), $type));
//                                     $t = $type;
//                                 }
                               
//                             }
                            
                            
                            if ($isNew){
                                $this->arrGeneral[$parentClass][] = array(
                                    "digestCaption" => $digestClass,
                                    "flavor" => $flavorClass,
                                    "x" => array("value" =>$x),
                                    "ivName" => array("value" => $ivname),
                                    "xcap" => array('xml:lang' => 'en', "value" => $xcap),
                                    "t" => array("value" => $t),
                                    "entityType" => $type
                                );
                            }
                         }
                    }
                }
            }
        }
        
        dumpDocsDbg("arrGeneral", $this->dcid, $this->arrGeneral);
    }
    
    function hasAandBInOrder($array, $elementA, $elementB) {
		if(empty($array)){
			return false;
		}
        $foundA = false;
        
        foreach ($array as $element) {
            if ($element === $elementA) {
                $foundA = true;
            } elseif ($element === $elementB && $foundA) {
                return true;
            }
        }
        
        return false;
    }
    
    public function getExistsTable($t_arr){
        $t_arr = is_string($t_arr) ? [$t_arr] : $t_arr;
        foreach($t_arr as $t){
            $table = $this->getTableFromUrl($t);
            if(!TablesInformation::isTableExists($this->mysqli, $table, $this->tables)){
                $arr = explode('#', $t);
                $parents_classes = $this->subClass2classes[$arr[1]] ?? [];
                foreach($parents_classes as $parent){
                    $table_parent = strtolower(str_replace('-', '_', $parent));
                    
                    if(TablesInformation::isTableExists($this->mysqli, $table_parent, $this->tables)){
                        return $parent;
                    }
                }
            }
            else{
                return $t;
            }
        }
        return null;
            
    }
    /**
     * @return $sql_array
     */
    public function getSql_array()
    {
        return $this->sql_array;
    }

    /**
     * @return $pro_anti_array
     */
    public function getPro_anti_array()
    {
        return $this->pro_anti_array;
    }

    public function createDigestJson2(){
        if ($this->arrGeneral){
            foreach ($this->arrGeneral as $k => $v){
                $menuChildren = array();
                $menuChildren_org = array();
                $distinctX = array(); //making sure that per parent concept there is only one instance on the same span
                foreach ($v as $value){
                    $xkey = unqualifyName($value['x']['value']);
                    if ($value['xcap']['xml:lang'] != 'en')
                        continue;
                    
                  
                    
                    $elementKeys = array();
                    $elementKeys_org = array();
                    $y = 0;
                    $elements = $this->getSpanElements($value['x']['value']);
                    $elementsLength = count($elements);
                    
                    if (! $elements || ! $elementsLength) {
                        // usually the KB instance has the iv:name and inserted to arrGeneral, but only the instace reference which refer to it is found in the digest object span list - so we have to find the instance reference
                        $objectNames = array(
                            $value['x']['value']
                        );
                        for ($depth = 0; $depth < 5; $depth ++) {
                            $elements = array();
                            foreach ($objectNames as $objectName) {
                                foreach ($this->getSpanElements($objectName) as $delement) {
                                    $elements[] = $delement;
                                }
                            }
                            if (! empty($elements)) {
                                break;
                            }
                            if (array_key_exists("iv:instance", $this->pred2inst["dig"]) && array_key_exists($objectName, $this->pred2inst["dig"]["iv:instance"])) {
                                $instanceReferences = $this->pred2inst["dig"]["iv:instance"][$objectName];
                            } else {
                                break;
                            }
                            if (empty($instanceReferences)) {
                                break;
                            }
                            
                            $objectNames = $this->getDigestSpanObjectNames($instanceReferences);
                        }
                        $elementsLength = count($elements);
                    }
                        
                    foreach ($elements as $delement) {
                        $deStart = $delement->getAttribute("start");
                        $deEnd = $delement->getAttribute("end");
                        if (array_key_exists($deStart, $this->arr_d)) {
                                // unfortunately 2 instances are found on the same span
                                if (array_key_exists($deStart, $distinctX) && $distinctX[$deStart] == $deEnd) {
                                    // only one instance per span per topic
                                } else {
                                    // use already existing key
                                    $elementKeys[] = $this->arr_d[$deStart][1];
                                }
                                $distinctX[$deStart] = $deEnd;
                                $elementKey = self::createSpanKey($deEnd);
                                $elementKey_org = $xkey;
                                if ($elementsLength > 1) {
                                    $elementKey_org .= $y ++;
                                }
                                if (count($this->arr_d[$deStart][2]) == 0) { //override the flavorless element
                                    $this->arr_d[$deStart] = array(
                                        $deEnd,
                                        $elementKey,
                                        $value['flavor']
                                    );
                                }
                                continue;
                        }
                        //elements are not on the same span
                        $elementKey = self::createSpanKey($deEnd);
                        $elementKey_org = $xkey;
                        if ($elementsLength > 1) {
                            $elementKey_org .= $y ++;
                        }
                        
                        $elementKeys[] = $elementKey;
                        $elementKeys_org[] = $elementKey_org;
                        $this->arr_d[$deStart] = array($deEnd, $elementKey, $value['flavor']);
                    }
                    
                    if (empty($elementKeys)) {
                        continue;
                    }
                    
                    unset($tmp);
                    $tmp = array(
                        'title' => $value['xcap']['value'],
                        'digestCaption' => $value['digestCaption'],
                        'flavor' => $value['flavor'],
                        'key' => $elementKeys[0],
                        'type' => $value['entityType'],
                        'ontUrl' => $value['ivName']['value'],
                        'entity-type' => $value['entityType']       
                    );
                    $menuChildren[] = &$tmp;
                    unset($tmp_org);
                    $tmp_org = array(
                        'title' => $value['xcap']['value'],
                        'key' => $elementKeys_org[0] ?? $elementKey_org,
                        'span_key' => $elementKeys[0],
                        'type' => $value['entityType'],
                        'ontUrl' => $value['ivName']['value'],
                        'entity-type' => $value['entityType']       
                    );
                    $menuChildren_org[] = &$tmp_org;
                    
                    if ($elementsLength == 1) {
                        continue;
                    }
                    
                    // multiple digest elements
                    $tmp['icon'] = false;
                    $tmp['children'] = array();
                    $tmp_org['children'] = array();
                    foreach ($elementKeys as $elementKey) {
                        $tmp['children'][] = array(
                            'title' => $value['xcap']['value'],
                            'key' => $elementKey,
                            'type' => $value['entityType'],
                            'ontUrl' => $value['ivName']['value'],
                            'entity-type' => $value['entityType']
                            
                        );
                        $tmp_org['children'][] = array(
                            'title' => $value['xcap']['value'],
                            'key' => $elementKey_org,
                            'span_key' => $elementKey,
                            'type' => $value['entityType'],
                            'ontUrl' => $value['ivName']['value'],
                            'entity-type' => $value['entityType']
                            
                        );
                    }
                }
                if(!empty($menuChildren)){
                    $this->digestJson[] = array('title' =>  $k, 'digestCaption' => $v[0]['digestCaption'],'flavor' => $v[0]['flavor'], 'key' => $elementKeys[0] ?? $elementKey, 'isFolder' => true, 'children' => $menuChildren, 'type' =>$value['entityType']);
                    $this->org_keys[] = array('title' =>  $k,  'key' => $k,  'span_key'=> $elementKeys[0] ?? $elementKey,'children' => $menuChildren_org, 'type' => $value['entityType']);
                    
                }
            }
        }
        dumpDocsDbg("digestJson2", $this->dcid, $this->digestJson);  
        dumpDocsDbg("org_keys2", $this->dcid, $this->org_keys);  
    }
    
    
    function createArrWord()
    {
        //detect URL in digest JSON
        $url_arr = [];
        $urlBool = false;
        
        foreach ($this->digestJson as $element){
            if(strtolower($element['title']) == 'url'){
                foreach($element["children"] as $child){
                    $url_arr[$child["key"]] = $child["title"];
                }
            }
        }
        
        $curr_arr_word = array();
        
        $word = "";
        $wordEndIndex = -1;
        $key = "";
        $prevChar = "";
        for ($utfIndex = 0, $charIndex = 0;
        ($character = $this->nextchar($this->text, $utfIndex)) !== false;
        $charIndex++)
        {
            //html escape special chars
            if ($character == "<") $htmlChar = "&lt;";
            elseif ($character == ">") $htmlChar = "&gt;";
            elseif ($character == "&") $htmlChar = "&amp;";
            elseif ($character == "\r" || ($character == "\n" && $prevChar != "\r")) $htmlChar = "<BR/>\n";
            else $htmlChar = $character;
            $prevChar = $character;
            if (array_key_exists($charIndex, $this->arr_d) && $wordEndIndex == -1 && count($this->arr_d[$charIndex][2]))
            {
                $wordEndIndex = $this->arr_d[$charIndex][0];
                $key = self::createSpanKey($wordEndIndex);
             
                $this->newText .= "<span id=\"$key\">";
                
                if(array_key_exists($key, $url_arr)){
                   $url = $url_arr[$key];
                   $this->newText .=  '<a href='. $url .' target="_blank">';
                   $urlBool = true;
                }
            }
            $this->newText .= $htmlChar;
            if ($wordEndIndex != -1)
            {
                $word .= $character;
                if ($wordEndIndex == $charIndex)
                {
                    if($urlBool){
                        $urlBool = false;
                        $this->newText .= "</a>";
                        
                    }
                    $this->newText .= "</span>";
                  
                    //choose the longest word (fix anaphora which refer to the same key, but almost always is smaller)
                    if (!array_key_exists($key, $curr_arr_word) || strlen($curr_arr_word[$key]) < strlen($word)){
                       
                        $curr_arr_word[$key] = $word;
                    }
                        $word= "";
                    
                        
                        $wordEndIndex = -1;
                }
            }
        }
        $this->arr_word = $curr_arr_word;
        dumpDocsDbg("arr_word", $this->dcid, $this->arr_word);
        dumpDocsDbg("newText", $this->dcid, $this->newText);
        
        
        
        
    }
    
    //temporary patch to prevent transliterated text in hashtags in digest Json
    public function patchHashTags(){
        $hastag_arr = [];
        
        foreach($this->digestJson as $key=>&$val){
            if($val['title'] == 'Twitter-hashtag'){
                foreach($val['children'] as &$child){
                    unset($unimportantName);
                    $orgText = $this->getInstanceOrgText($child, $unimportantName);
                    $modified = strtolower($orgText);
                    $modified = str_replace(["-","_"], " ", $modified);
                    $hastag_arr[] = $modified ?? $child['title'];
                }
                $val['children'] = [];
                $id = 0;
                foreach($hastag_arr as $title){
                    $val['children'][] = ["id"=>$id,'title'=>$title, 'type'=>"http://www.intuview.com/ontology#Twitter-hashtag", "entity-type"=>"http://www.intuview.com/ontology#Twitter-hashtag"];
                    $id++;
                }
                break;
            }
        }
    }
    
    public function fillSentiment()
    {
        $settings_class =  new Settings($this->mysqli, $this->username);        
        $flavors_arr =  $settings_class->getSettings("be", "getFlavors");
        foreach($flavors_arr as $item){
            $flavors[] = $item["name"];
        }  
        
        foreach (array("anti" => "negative", "pro" => "positive") as $sent => $neg_pos)
        {
            $span_arr_tracker = [];
            foreach ($this->pred2inst as $doa => $dig_or_agg)
            {
                
                if (array_key_exists("iv:sentiment-".$sent,$dig_or_agg) && $dig_or_agg["iv:sentiment-" . $sent]) {
//                     $sent_caption = $sent == 'anti' ? "Negative" : "Positive";
//                     $sent_title = "$sent_caption Sentiment";
                    
//                     if($_REQUEST['getmetadata']){
//                         if(empty( $this->meta_arr["intuview_data"])){
//                             $this->meta_arr["intuview_data"] = [];
//                         }
//                         if(empty( $this->meta_arr["intuview_data"]["concepts"])){
//                             $this->meta_arr["intuview_data"]["concepts"] = [];
//                         }     
//                         if(empty($this->meta_arr["intuview_data"]["concepts"][$sent_title])){
//                             $this->meta_arr["intuview_data"]["concepts"][$sent_title] = [];
//                         }
//                     }
                    
//                     $sentElement = ['title' => $sent_title, 'digestCaption' => array_fill(0, count($flavors), $sent_title), 'key' => $sent_title, 'isFolder'=>'true',
//                         'children' => [], 'flavor'=> $flavors, 'type' => $sent, 'entity-type' =>$sent_title];
                    
                    
                    foreach ($dig_or_agg["iv:sentiment-" . $sent] as $sentInstQName => $instances) {
                        $sentInstType = $this->getSentInstType($sentInstQName);
                        
                        $sentInstCaption = $this->qname2node["dig"][$sentInstQName]["iv:caption-base"] ?? $this->qname2node["dig"][$sentInstQName]["iv:caption"] ?? null;
                        if (is_array($sentInstCaption)){
                            $sentInstCaption = $sentInstCaption["en"] ?? null;
                        }
        

//                         if($_REQUEST['getmetadata']){
//                             $this->meta_arr["intuview_data"]["concepts"][$sent_title][$sentInstCaption] = ['name'=>$sentInstCaption];
//                         }
//                         $sentElement['children'][] = ['title'=>$sentInstCaption, 'key'=>"", 'ontUrl'=>$sentInstQName,
//                             'type'=>$sentInstType, 'entity-type'=>$sentInstType];
                        
                        foreach ($instances as $instance) {
                            $instCaption = $instance["iv:caption"] ?? null;
                            
                            if (is_array($instCaption)){
                                if(!array_key_exists("en", $instCaption)){
                                    continue;
                                }
                                $instCaption = $instCaption["en"];
                            }
                            
                            $instTypes = [];
                            $insttypearray = $instance["rdf:type"];
                            if (!is_array($insttypearray)) {
                                $insttypearray = [$insttypearray];
                            }
                            foreach ($insttypearray as $multype) {
                                $rdfType = unqualifyName($multype);
                                if (array_key_exists($rdfType, $this->annotElementsParentClasses)){
                                    $instTypes = $this->annotElementsParentClasses[$rdfType];
                                }
                                else{
                                    write_to_log("TRACE", "$rdfType is not found in parent classes");
                                }
                            }
                            
                            
                            if (array_key_exists("iv:entity-type", $instance)){
                                $instTypes[] = unqualifyName($instance["iv:entity-type"]);
                            }
                            $instTypes = array_unique($instTypes, SORT_REGULAR);
                            
                            foreach ($instTypes as $conceptName) {
                                if(is_array($conceptName)){
                                    continue;
                                }
                                $conceptName = TablesInformation::getTableName($conceptName);
                                //personStrippedName gets filled only in table2valuemaker which called after..
                                if ($conceptName == "person_object" && array_key_exists($instCaption, $this->personStrippedName)){
                                    $instCaption = $this->personStrippedName[$instCaption];
                                }
                                
                                $this->conceptsSentiment[$conceptName][$instCaption][$sent][] = $sentInstCaption;
                                $this->conceptsSentiment[$conceptName][$instCaption][$neg_pos][] = $sentInstType;
                            }
                            if(!empty($instCaption)){
                                 $this->aggregated_sentiment[$sent][] = $instCaption;
                            }
                            $elements = $this->getSpanElements($sentInstQName);
              
                        }
                    }
//                     $this->digestJson[] = $sentElement;
                    

                }
            }
        }
        
        dumpDocsDbg("conceptsSentiment", $this->dcid, $this->conceptsSentiment);
        
    }
    
    public function table2valuesMaker() { 
        global $listPageColumn;
        global $organizationARC_prefix;
        global $organizationARC;
        global $entitiesArray;
        $person_index = 0;
        $concept_indexes_alert = getAlertSettingsProp($this->mysqli, 'Concept_alert_list');

        $alerts_concepts_arr = [];
        if(!empty(trim($concept_indexes_alert))){
            $concept_indexes_alert_arr = explode(',', $concept_indexes_alert);
            foreach($concept_indexes_alert_arr as $id){
                $alert_concept = getConceptFromId($this->mysqli, $id);
                if(!empty($alert_concept)){
                    $alerts_concepts_arr[] = $alert_concept;
                }
            }
        }
        
        $concept_instances_alert = getAlertSettingsProp($this->mysqli, 'concepts_instances') ;
        $concept_instances_alert = json_decode($concept_instances_alert, true) ?? [];
        foreach($concept_instances_alert as $concept=>&$element){
            foreach ($element as &$pair){
                $pair = $pair['ont'] ?? $pair['label'];
            }
        }
        foreach ($this->org_keys as $concept)
        {   
         
            $conceptName = strtolower(str_replace('-','_',$concept['key']));
            $tableName = $conceptName;
 
            //if table name is under alers settings - add 
            if(in_array($conceptName, $alerts_concepts_arr)){
                $this->alertsArray['concepts'][$conceptName] = [];
            }
            
            
            //Checks if concept is in the tables, or _version table        
            if (in_array($conceptName, $this->sqlTables))
            {
                foreach ($concept['children'] as $value){
                    //reset concept_id & concept_id_source indexes
                    $concept_id = $this->getConceptId($conceptName);
                    $concept_id_source = $concept_id;
                    
                    $ont_name = "";
                    if (stristr($value['key'],"dynamic-inst-ref-") && (stristr($value['key'],"-object") == false)){
                        $str = explode('dynamic-inst-ref-',$value['key']);
                        $s = explode('-',$str[1]);
                        $i = 0;
                        for ($i = 0; $i < (sizeof($s) - 1); $i++){
                            if ($i == 0){
                                $ont_name .= $s[$i];
                            }
                            else{
                                $ont_name .= "-" . $s[$i];
                            }
                        }
                    }
                    else{
                        if(empty($value['title'])){
                            continue;
                        }
                        $ont_name = str_replace(" ", "-" , $value['title']);
                        $ont_name = str_replace("'" ,"",$ont_name);
                        $ont_name = str_replace('"' ,"",$ont_name);
                    }
                    $ont_name = toDbString($this->mysqli, $ont_name);
                    
         
                    $unescapedName = $value['title'];
                    $valueType = [];
                    if( !is_array($value['type']) ) {
                        $valueType[] = str_replace('http://www.intuview.com/ontology#', '', $value['type']);
                    }
                    else{
                        foreach ($value['type'] as $vt){
                            $valueType[] = str_replace('http://www.intuview.com/ontology#', '', $vt);
                        }
                    }
                    
                    $e = join(",", $valueType);
                    $type_c = toDbString($this->mysqli, $e);
                    $name = toDbString($this->mysqli, $unescapedName);

                    //check aggregated and crossentitie
                    $aggCeArr = $this->checkInAggOrCross($conceptName, $name);
                    $name_agg = $aggCeArr["name_agg"];
                    $destConcept = $aggCeArr["destConcept"];
                    
                    //check ALERTS 
                    //if table name is under alers settings - add
                    if(array_key_exists($conceptName, $this->alertsArray['concepts'] ?? []) && !in_array($name, $this->alertsArray['concepts'][$conceptName])){
                        $this->alertsArray['concepts'][$conceptName][] = $name;
                    }
                    
                    //if table name is under instances alerts
                    if(array_key_exists($conceptName, $concept_instances_alert) && 
                        (in_array($ont_name, $concept_instances_alert[$conceptName]) || 
                            in_array($name, $concept_instances_alert[$conceptName]))){
                        $this->alertsArray['instances'][$conceptName][] = $name;
                    }
                    
                    
                    $docKey = toDbString($this->mysqli, $value['key']);
                    unset($unimportantName);
                    $orgText = $this->getInstanceOrgText($value, $unimportantName);
                    if (strlen($orgText) > 1){
                        $this->orgTextKeyArr[$value['key']] = $orgText;
                        if(is_array($value['title'])){
                            $value['title'] = reset($value['title']);
                        }
                        $this->orgTextValueArr[$value['title']] = $orgText;
                        
                    }
                    $orgText = toDbString($this->mysqli, $orgText);

                    //arabic hashtags have buck-walter caption, so we need to replace it with orgText
                    if ($conceptName == "twitter_hashtag"){
                        $name = $orgText;
                    }
                    
                    if ($conceptName == "person_object"){
                        $name_split = explode("(" , $value['title']);
                        $unescapedName = trim($name_split[0]);
                       
                        $this->personKey2idName[$value["key"]]["name"] = $unescapedName;
                        //remember the stripped name
                        $this->personStrippedName[$value['title']] = $unescapedName;
                        $name = toDbString($this->mysqli, $unescapedName);
                        if (sizeof($name_split) > 1){
                            $modifier = "(" . $name_split[1];
                        }
                        else{
                            $modifier = "";
                        }
                    }
                    $name = trim($name);
              
                    
                    //prevent duplications
                    if (array_key_exists($tableName, $this->concept2name) && array_key_exists($name, $this->concept2name[$tableName]))
                    {
                        write_to_log("WARNING", "$name already exists in table $tableName in doc $this->dcid");
                        continue;
                    }
                    
                    $this->concept2name[$tableName][$name] = true;
                    
                    if (array_key_exists("children", $value)){
                        $count_childern = max(1,sizeof($value['children']));
                    }
                    else{
                        $count_childern = 1;
                    }
                    
                    $getMetadataIndexSlots = null;
                    if($_REQUEST["getmetadata"] && $conceptName !== 'concept'){
                        if(empty( $this->meta_arr["intuview_data"])){
                            $this->meta_arr["intuview_data"] = [];
                        }
                        
                        if(empty($this->meta_arr["intuview_data"]["concepts"][$conceptName])){
                            $this->meta_arr["intuview_data"]["concepts"][$conceptName] = [];
                        }
                        
                        $element_intuview_data = ["name" => $name, "ont" => $ont_name, "count" => $count_childern]; 
                        
                        //check coordinates for intuview-data api
                        if($conceptName=='place_object'){
                            $sql = "SELECT coordinates FROM coordinates WHERE location='$ont_name'";
                            $res = sqlQuery($this->mysqli, $sql);
                            if($res){
                                $obj = $res->fetch_object();
                                if(!empty($obj)){
                                    $element_intuview_data['coord'] = $obj->coordinates;
                                }
                            }
                        }
                        $this->meta_arr["intuview_data"]["concepts"][$conceptName][$name] = $element_intuview_data;
                        
                        $getMetadataIndexSlots = count($this->meta_arr["intuview_data"]["concepts"][$conceptName])-1;
                    }
                    
                    $this->slotDataHandlerT2V($conceptName, $value, $name, $count_childern, $modifier ?? null, $getMetadataIndexSlots);
                    
                
                    $url = array_key_exists("ontUrl", $value)?unqualifyName($value['ontUrl']):"";
                    
                    if (strlen($name) > 1){
                        $name = trim($name);
                        
                        $sent_arr = $this->handleSentimentsT2V($value, $conceptName, $name, $tableName, $unescapedName);
                        
                        $curr_pro = $sent_arr['pro'];
                        $curr_positive = $sent_arr['positive'];
                        $curr_anti = $sent_arr['anti'];
                        $curr_negative = $sent_arr['negative'];
                 
                        //A - person_object
                        if ($conceptName == "person_object" || $conceptName == "organizational_identity")
                        {
                            
                            if(!empty($destConcept)){
                                $concept_id = $this->getConceptId($destConcept);
                            }
   
                            $modifierEsc = toDbString($this->mysqli, $modifier ?? "");
                            $name_c = self::getCleanName($unescapedName, $organizationARC, $organizationARC_prefix);

                            $table_columns = ['concept_type', 'concept_type_source', 'docId','name_version','clean_name','modifier','ontName','ontUrl','docKey','username','occurrences','type','orgText_version', 'pro', 'anti', 'positive', 'negative'];
                            $table_values = [$concept_id, $concept_id_source, $this->dcid,"\"$name\"","\"$name_c\"","\"$modifierEsc\"","\"$ont_name\"","\"$url\"","\"$docKey\"","\"$this->username\"","\"$count_childern\"","\"$type_c\"","\"$orgText\"", "\"$curr_pro\"", "\"$curr_anti\"", "\"$curr_positive\"", "\"$curr_negative\""];                       
                            
                            if(!empty($name_agg)){
                                $table_columns[] = "names_agg";
                                $table_values[] = "\"$name_agg\"";
                            }
                            $table_columns = "(".implode(',', $table_columns).")";
                            $table_values = "(".implode(',', $table_values).")";
                            if($conceptName == "person_object"){
                                if(!in_array($name, $this->persons_names)){
                                    $person_data = $this->addNameToWL($name, $person_index);
                                    if(!empty(trim($this->entities_alert_wl))){                                    
                                        $person_data = $this->searchPersonInWL($name);
                                    }
                                    $this->persons_data[] = $person_data;
                                    $this->persons_names[] = $name;
                                  
                                    $person_index++;                                   
                                }
                                else{
                                    write_to_log("ERROR", "person already in WL");
                                }
                            }
                            $this->table2values[$conceptName][$table_columns]["keys"][] = $value["key"];
                            $this->table2values[$conceptName][$table_columns]["values"][] = $table_values;
//                             $this->solr_array[$conceptName][$this->dcid][] = array("name"=>$name,"orgText"=>$orgText);
                            
                        }

                        //C - rest
                        else {
                            //place_object, events
                            if(in_array($conceptName, array("place_object","event"))){
                                //add manage entities name if exists
                                if(!empty($destConcept)){
                                    $concept_id = $this->getConceptId($destConcept);
                                }
//                                 $this->solr_array[$conceptName][$this->dcid][] = array("name"=>$name,"orgText"=>$orgText);
                            }

                            
                            $sqlColumns = ["concept_type", "concept_type_source",  "docId","name_version","ontName","ontUrl","docKey","username","occurrences","type", "orgText_version", "pro","anti","positive","negative"];
                            $sqlValues = [$concept_id, $concept_id_source, $this->dcid,"\"$name\"","\"$ont_name\"","\"$url\"","\"$docKey\"","\"$this->username\"","\"$count_childern\"","\"$type_c\"", "\"$orgText\"", "\"$curr_pro\"", "\"$curr_anti\"", "\"$curr_positive\"", "\"$curr_negative\""];
                            
                           
                            //C1 - dates
                            if (stristr($conceptName,"date")){
                                $date = $this->handleDateT2V($docKey);
                                if(!empty($date)){
                                    $sqlColumns[] = "date";
                                    $sqlValues[] = "\"$date\"";
                                }
                            }
                            
                            //C2 - Locations
                            elseif ((stristr($conceptName,"place") || stristr($conceptName,"City") || stristr($conceptName,"Province")  || stristr($conceptName,"Neighborhood")) && strlen($ont_name)>1){
                                    $loc_res = $this->handleLocationsT2V($conceptName, $ont_name, $value, $sqlColumns, $sqlValues);
                                    $sqlColumns = array_merge($sqlColumns, $loc_res['cols']);
                                    $sqlValues = array_merge($sqlValues, $loc_res['vals']); ;
                            }
                          
                            
                            $sqlColumns = "(".implode(',', $sqlColumns).")";
                            $sqlValues = "(".implode(',', $sqlValues).")";
                            
                            
                            $this->table2values[$conceptName][$sqlColumns]["keys"][] = $value["key"];
                            $this->table2values[$conceptName][$sqlColumns]["values"][] = $sqlValues;
                        }
                        
                        
                        $this->key2idConcept[$value["key"]] = array("table" => $conceptName);
                    }
                    
                }
            }
           
            //if the concept has org parent - we need the clean name 
            if(isset($this->subClass2classes[$concept['key']]) && in_array('Organizational-identity',$this->subClass2classes[$concept['key']])){
                 $clean_name = self::getCleanName($unescapedName, $organizationARC, $organizationARC_prefix);
            }
            //handle parent classes - insert them into table2values
            $this->parentClassHandler($conceptName, $concept, $name, $clean_name ?? null, $alerts_concepts_arr);
        }
        if(isset($this->sentimentArr)){
            dumpDocsDbg("sentimentArr", $this->dcid, $this->sentimentArr);
        }
        dumpDocsDbg("table2values", $this->dcid, $this->table2values);
    }
    
    function addNameToWl($name,  $index){
        require_once '../iventitymatcherhtm/utilities/na_utils.php';
        require_once '../iventitymatcherhtm/utilities/sql_utilies.php';
        require_once '../iventitymatcherhtm/utilities/utilities.php';
        
        $person_ref = $this->dcid."_".$index;
        
        global $databaseName;
        $databaseName_str = strtolower($databaseName);

        $sql_final = [];
        $person_data = fetchPersonData($this->ivEntityMatcherService, $name,  true, true);
//         $person_data = fetchPersonData($this->ivEntityMatcherService, $name, null, true);
        
        insertPersonDataToDB($this->wl_mysqli, 1, $this->sql_final, $this->sql_rules_final, $index, "wl_$databaseName_str", $person_ref, $person_data, $name, $this->rules_json);
        return $person_data;
    }
    
    
    function searchPersonInWL($name){
        require_once '../iventitymatcherhtm/utilities/na_utils.php';
        require_once '../iventitymatcherhtm/utilities/sql_utilies.php';
        require_once '../iventitymatcherhtm/utilities/utilities.php';
        require_once '..'.DIRECTORY_SEPARATOR.'iventitymatcherhtm'.DIRECTORY_SEPARATOR.'compareWLClass.php';
        
        $WLclass = new CompareWLClass($this->mysqli,"name");
        
        $response = $WLclass->searchPersonInList($name, $this->entities_alert_wl, true);
        $found_names = [];
        
        foreach($response as $na){
            if(!empty($na["matching_results"])){
                
                if(!array_key_exists('entities', $this->alertsArray)){
                    $this->alertsArray['entities'] = [];
                }
                if(!array_key_exists('persons', $this->alertsArray['entities'])){
                    $this->alertsArray['entities']['persons'] = [];
                }
                
                foreach ($na["matching_results"] as $rule=>$arr){
                    $rule_str = ucwords(str_replace('_', ' ', $rule));
                    foreach($arr as $data){
                        $inner_name = $data["Name"];
                        if(!in_array($inner_name,$found_names[$rule] ?? [])){
                            $found_names[$rule][] = $inner_name;
                            
                            if(!array_key_exists($rule_str, $this->alertsArray['entities']['persons'])){
                                $this->alertsArray['entities']['persons'][$rule_str] = [];
                            }
                            
                            $this->alertsArray['entities']['persons'][$rule_str][] = ['name'=>$inner_name, 'confidence'=>$data["score"]];
                        }
                    }
                    
                }
            }
        }

    }
 
   
    function updateAndSendNotifications(){
        if(empty($this->alertsArray)){
            write_to_log("INFO", "No Alerts found in the document");
            return;
        }
        
        /**
         * 1 - update database
         */
        $entities_alerts_compress_sql = "";
        if(!empty($this->alertsArray['entities'])){
            if(!empty($this->alertsArray['entities']['persons'])){
                $entities_alerts_str = json_encode($this->alertsArray['entities']['persons'],true);
                $entities_alerts_compress = base64_encode(gzcompress($entities_alerts_str,9));
                $entities_alerts_compress_sql = $this->mysqli->real_escape_string($entities_alerts_compress);
            }
        }
        
        $concepts_alerts_compress_sql = "";
        if(!empty($this->alertsArray['concepts'])){
            $concepts_alerts_str = json_encode($this->alertsArray['concepts'],true);
            $concepts_alerts_compress = base64_encode(gzcompress($concepts_alerts_str,9));
            $concepts_alerts_compress_sql = $this->mysqli->real_escape_string($concepts_alerts_compress);
        }
        
        $instances_alerts_compress_sql = "";
        if(!empty($this->alertsArray['instances'])){
            $instances_alerts_str = json_encode($this->alertsArray['instances'],true);
            $instances_alerts_compress = base64_encode(gzcompress($instances_alerts_str,9));
            $instances_alerts_compress_sql = $this->mysqli->real_escape_string($instances_alerts_compress);
        }
        
        $keywords_alerts_compress_sql = "";
        if(!empty($this->alertsArray['keywords'])){
            $keywords_alerts_str = implode(",", $this->alertsArray['keywords']);//json_encode($this->alertsArray['keywords'],true);
            $keywords_alerts_compress = base64_encode(gzcompress($keywords_alerts_str,9));
            $keywords_alerts_compress_sql = $this->mysqli->real_escape_string($keywords_alerts_compress);
        }
        
        $this->doc_username = !empty($this->doc_username) ? $this->doc_username["name"].",".$this->doc_username["id"] : "";
 
        $sql = "INSERT INTO alerts (docId, doc_username, entities_alerts, concepts_alerts, instances_alerts, keywords_alerts, username,reviewed) VALUE($this->dcid, '$this->doc_username', '$entities_alerts_compress_sql', '$concepts_alerts_compress_sql', '$instances_alerts_compress_sql','$keywords_alerts_compress_sql', \"$this->username\", 0)";
        $res = sqlQuery($this->mysqli, $sql);
        
        /**
         * 2 - send email notifications
         */
//        $this->sendAlertsEmail();
    }
    
    /**
     * sendAlertsEmail - send the alerts_arr as email to reciever
     */
    function sendAlertsEmail(){
        $sender = getSystemSettingsProp($this->mysqli, "alert_sender");
        $sender_pass = getSystemSettingsProp($this->mysqli, "sender_pass");
        
        
        $reciever = getSystemSettingsProp($this->mysqli, "alert_reciever");
        
        //if no alerts were found OR no sender email definded || no reciver email - skip
        if(empty($sender) || empty($reciever)){
            return;
        }
        write_to_log("TRACE", "Send notification");
        
        
        //Load Composer's autoloader
        require 'vendor/autoload.php';
        global $databaseName;
        $message_names = [];
        
        $message = "Project: $databaseName<BR>";
        $message .= "Document: $this->dcid<BR><BR>";
        $message .= "================<BR><BR>";
        foreach($this->alertsArray['persons'] as $rule=>$elements_arr){
            $message .= $rule.":<BR>";
            foreach($elements_arr as $element){
                $name = trim($element['name']);
                $message .= "Name: $name & confidence: $element[confidence]<BR>";
            }
        }
        
        //Create an instance; passing `true` enables exceptions
        $mail = new PHPMailer(true);
        $mail->IsSMTP();                                        // telling the class to use SMTP
        $mail->SMTPDebug    = 1;                                // enables SMTP debug information (for testing)
        $mail->SMTPAuth     = true;                             // enable SMTP authentication
        $mail->SMTPSecure   = "tls";                            // sets the prefix to the servier
        $mail->Host         = "smtp.gmail.com";                 // sets GMAIL as the SMTP server
        $mail->Port         = 587;                              // set the SMTP port for the GMAIL server
        
        $mail->Username     = $sender  ;           // GMAIL username
        $mail->Password     = $sender_pass ;           // GMAIL password
        
        $mail->SetFrom($sender, 'Intuview');
        $mail->Subject = "Intuview Alert";
        $mail->MsgHTML($message);
        $mail->AddAddress($reciever, "Shmuel");
        
        // $mail->AddAttachment("images/phpmailer.gif");        // attachment
        // $mail->AddAttachment("images/phpmailer_mini.gif");   // attachment
        
        if(!$mail->Send()) {
            echo "Mailer Error: " . $mail->ErrorInfo;
        }
        else {
            echo "Message sent!";
        }
    }
    
    /*
     * gets the id of the concept int the parent tables 
     */
    function getConceptId($conceptName){
        if(empty($conceptName)){
            return null;
        }
        $sql = "SELECT id FROM concepts_indexes WHERE concept='$conceptName'";
        $res = sqlQuery($this->mysqli, $sql);
        if ($res!=FALSE){
            $res = $res->fetch_object();
            if(empty($res)){
                return null;
            }
            return $res->id;
        }
        return null;
    }
    
    public function handleDateT2V($docKey){
        //extract date-time slot form date instance
        $dateQname = qualifyName($docKey, $this->metaParameter["terminal_system_id"]);
        if (array_key_exists($dateQname, $this->qname2node["agg"])){
            $date =$this->qname2node["agg"]["iv:date-time"] ?? null;
        }
        elseif (array_key_exists($dateQname, $this->qname2node["dig"])){
            $date = $this->qname2node["dig"][$dateQname]["iv:date-time"];
        }
        else{
            write_to_log("WARNING", "date not found in qname2node: $dateQname");
        }
        
        //convert date from fromat d/m/Y to Y-m-d
        if (!empty($date))
        {
            $dmy = explode("/", $date);
            if ($dmy && count($dmy) == 3)
            {
                $day = $dmy[0];
                $month = $dmy[1];
                $year = $dmy[2];
                if ($day == "00"){
                    $day = "01";
                }
                if ($year == "0000"){
                    $year = "1970";
                }
                $date = "$year-$month-$day";
                return $date;
                
            }
        }
        
    }
    
    /**
     * handleSentiments in table2values
     * @param unknown $value
     * @param unknown $conceptName
     * @param unknown $name
     * @param unknown $tableName
     * @param unknown $unescapedName
     * @return NULL[]
     */
    function handleSentimentsT2V($value, $conceptName, $name, $tableName, $unescapedName){
        $sent_arr = array();
        //handle sentiment
        if(is_array($value['title'])){
            $value['title'] = $value['title'][0];
        }
        if(is_array($unescapedName)){
            $unescapedName = reset($unescapedName);
        }
        
        if(!is_string($tableName) && !is_int($tableName)){
            write_to_log("TRACE", "Issue with table: $tableName");
            return;
        }
        
        if (array_key_exists($tableName, $this->conceptsSentiment) && array_key_exists($unescapedName, $this->conceptsSentiment[$tableName])){
            $instSent = $this->conceptsSentiment[$tableName][$unescapedName];
        }
        elseif (array_key_exists($tableName, $this->conceptsSentiment) && array_key_exists($value['title'], $this->conceptsSentiment[$tableName])){
            $instSent = $this->conceptsSentiment[$tableName][$value['title']];
        }
        else{
            $instSent = [];
        }
        
        $sent_arr = array('pro' => NULL,'positive' => NULL, 'anti'=> NULL, 'negative' => NULL);
        foreach ($instSent as $colName => $vals)
        {
            $this->sentimentArr[$conceptName][$name][$colName] = array_unique($vals);
            $sent_arr[$colName] = toDbString($this->mysqli, implode(",", array_unique($vals)));
            
            if($_REQUEST["getmetadata"] && in_array($colName, ['anti','pro'])){
                foreach(array_unique($vals) as $sentiment){
                    $this->meta_arr["intuview_data"]["concepts"][$conceptName][$name]['sentiment'][$colName][] = $sentiment;
                }
            }
            
            
        }
        
        return $sent_arr;
    }
    
    
    /**
     * halper for table2valuesmaker - handle the locations tables
     * @param unknown $conceptName
     * @param unknown $ont_name
     * @param unknown $value
     * @param unknown $sqlColumns
     * @param unknown $sqlValues
     */
    function handleLocationsT2V($conceptName, $ont_name, $value){
        $sqlValues = [];
        $sqlColumns = [];
        $country = $countryQname = "";
        $province = $provinceQname = "";
        $city = $cityQname = "";
        $neighborhood = $neighborhoodQname = "";
        //get country caption from slot _Country
        $qualifiedPlaceName = qualifyName($value["key"], $this->metaParameter["terminal_system_id"]);
       
        foreach (array("agg", "dig") as $aggDig){
            if (array_key_exists($qualifiedPlaceName, $this->qname2node[$aggDig])){
                $countryQname = $this->qname2node[$aggDig][$qualifiedPlaceName]["iv:country"]??null;
                $provinceQname = $this->qname2node[$aggDig][$qualifiedPlaceName]["iv:state-or-province"]??null;
                $cityQname = $this->qname2node[$aggDig][$qualifiedPlaceName]["iv:city"]?? null;
                $neighborhoodQname = $this->qname2node[$aggDig][$qualifiedPlaceName]["iv:neighborhood"]?? null; 
            }
        }
        
        //country
        if (!empty($countryQname)){
            if(is_array($countryQname)){
                $country_arr = [];
                foreach($countryQname as $qname){
                    if(array_key_exists($qname, $this->qname2node["dig"])){
                        $country_arr[] = $this->qname2node["dig"][$qname]["iv:caption"]["en"];
                    }
                }
                $country = implode(",", $country_arr);
                
            }
            else{
                if(array_key_exists($countryQname, $this->qname2node["dig"])){
                    $country = $this->qname2node["dig"][$countryQname]["iv:caption"]["en"];
                }
            }
        }
        
        //province
        if (!empty($provinceQname)){
            if(is_array($provinceQname)){
                $province_arr = [];
                foreach($provinceQname as $qname){
                    if(array_key_exists($qname, $this->qname2node["dig"])){
                        $province_arr[] = $this->qname2node["dig"][$qname]["iv:caption"]["en"];
                    }
                }
                $province = implode(",", $province_arr);
                
            }
            else{
                if(array_key_exists($provinceQname, $this->qname2node["dig"])){
                    $province = $this->qname2node["dig"][$provinceQname]["iv:caption"]["en"];
                }
            }
        }
        
        //city
        if (!empty($cityQname)){
            if(is_array($cityQname)){
                $city_arr = [];
                foreach($cityQname as $qname){
                    if(array_key_exists($qname, $this->qname2node["dig"])){
                        $city_arr[] = $this->qname2node["dig"][$qname]["iv:caption"]["en"];
                    }
                }
                $city = implode(",", $city_arr);
                
            }
            else{
                if(array_key_exists($cityQname, $this->qname2node["dig"])){
                    $city = $this->qname2node["dig"][$cityQname]["iv:caption"]["en"];
                }
            }
        }
        
        //neighborhood
        if (!empty($neighborhoodQname)){
            if(is_array($$neighborhoodQname)){
                $neighborhood_arr = [];
                foreach($neighborhoodQname as $qname){
                    if(array_key_exists($qname, $this->qname2node["dig"])){
                        $neighborhood_arr[] = $this->qname2node["dig"][$qname]["iv:caption"]["en"];
                    }
                }
                $neighborhood = implode(",", $neighborhood_arr);
            }
            else{
                if(array_key_exists($neighborhoodQname, $this->qname2node["dig"])){
                    $neighborhood = $this->qname2node["dig"][$neighborhoodQname]["iv:caption"]["en"];
                }
            }
        }
        
     
   
        
        if (empty($country))
        {
            $e = explode("-",$ont_name);
            if (sizeof($e) == 3){
                $country = $e[1];
            }
        }
        if (!empty($country))
        {
            $sqlColumns[] = "country";
            $sqlValues[] = '"'.toDbString($this->mysqli, $country).'"';//"\"$country\"";
        }
        if (!empty($province))
        {
            $sqlColumns[] = "province";
            $sqlValues[] = '"'.toDbString($this->mysqli, $province).'"';//"\"$province\""; 
        }
        if (!empty($city))
        {
            $sqlColumns[] = "city";
            $sqlValues[] = '"'.toDbString($this->mysqli, $city).'"';//"\"$city\"";      
        }
        if (!empty($neighborhood))
        {
            $sqlColumns[] = "neighborhood";            
            $sqlValues[] = '"'.toDbString($this->mysqli, $neighborhood).'"';//"\"$neighborhood\"";        
        }
        return array("cols"=>$sqlColumns,
                     "vals"=>$sqlValues
        );
    }
    
    
    /**
     * slotDataHandler($conceptName, $value, $modifier)
     * @param unknown $conceptName
     * @param unknown $value
     * @param unknown $modifier
     */
    public function slotDataHandlerT2V($conceptName, $value, $name, $count_childern, $modifier, $slotDataHandlerT2V){
        global $slotToCaption;
        
        foreach (array('dig', 'agg') as $section){
            $qname2node_exists = array_key_exists("http://www.intuview.com/doc$this->dcid#".$value['key'],$this->qname2node[$section]);
            $in_concepts_bool =  ($conceptName == "aya" || $conceptName == "place_object" || $conceptName == "organizational_identity" || $conceptName == "person_object");
            //removed $in_concepts_bool - it filtered out everything that is not aya,place,org,person
            if( $qname2node_exists  ) {
                $this->emotion_concept++;
                $slots = $this->qname2node[$section]["http://www.intuview.com/doc$this->dcid#".$value['key']];
                if( !array_key_exists("iv:caption", $slots) && array_key_exists("iv:transliteration-name",$slots) )
                {//compensate for lack of caption in instances like Aya
                    $slots["iv:caption"] = $slots["iv:transliteration-name"];
                }
                $slotdataColumns = ["docId", "conceptUrl"];//"(docId,conceptName,conceptUrl %MORE_COLS%)";
                $slotdataValues = [$this->dcid, "\"$value[key]\""];//"($this->dcid,\"$conceptName\",\"$value[key]\" %MORE_VALS%)";

                if ($conceptName == "person_object"){
                    $name = trim($name);
                    $slotdataColumns[] = 'caption';
                    $slotdataValues[] = '"'.toDbString($this->mysqli, $value['title']).'"';
                    if (strlen($modifier) > 1){
                        $slotdataColumns[] = "modifier";
                        $slotdataValues[] = '"'.toDbString($this->mysqli, $modifier).'"';
                    }
                    
                }
                
                foreach ($slots as $k_s => $v_s){
                    $e = explode("iv:" , $k_s);
                    
                    //special care for name alternative slots
                    if($k_s == 'iv:name-alternatives' && is_string($v_s)){
                        $na_slots = $this->qname2node["dig"][$v_s];
                    }
                    //use specific type from slots as conceptname
                    if($k_s == 'rdf:type'){
                        $slotdataColumns[] = "conceptName";
                        $slotdataValues[] = "'".$this->getTableFromUrl($v_s)."'";      
                    }
                    
                    if (count($e) > 1 && (array_key_exists($e[1],$slotToCaption))){
                        $key_s = str_replace("-", "_", $e[1]);
                        $k_t = $slotToCaption[$e[1]]['label'] ?? $slotToCaption[$name]['label'];
                        $v_t = slotValueToCaption($v_s, $this->qname2node);
                        
                        if ($conceptName !== "person_object" || ($k_t != "Name" && $k_t != "Modifier")){
                            $slotdataColumns[] = $key_s;
                            $slotdataValues[] = '"'.toDbString($this->mysqli, $v_t).'"';
                        }
                        if($_REQUEST['getmetadata']){
                            if($k_t !== "Name"){
                                $this->meta_arr["intuview_data"]["concepts"][$conceptName][$name]['slots'][] = ["prop"=>$k_t, "name"=>$v_t];
                            }
                        }
                    }
                  
                }
                //special care for name alternative fields - override (or add if not exists) over the slotdata of the entity
                if(!empty($na_slots)){
                    foreach($na_slots as $na_k=>$na_v){
                        $e = explode("iv:" , $na_k);
                        $key_s = count($e) > 1 ? str_replace("-", "_", $e[1]) : null;
                        $k_t = count($e) > 1 ? ($slotToCaption[$e[1]]['label'] ?? null) : (($slotToCaption[$name]['label'] ?? null) ?? null );
                        $v_t = slotValueToCaption($na_v, $this->qname2node);
                                                
                        if (count($e) > 1 && (array_key_exists($e[1],$slotToCaption))){
                            $na_v_c = slotValueToCaption($na_v, $this->qname2node);
                            $slot_field = $e[1];
                            $index = array_search($slot_field, $slotdataColumns);
                            if(!empty($index)){
                                $slotdataValues[$index] = '"'.toDbString($this->mysqli, $na_v_c).'"'; //override the value if exists
                            }
                            elseif ($conceptName !== "person_object" || ($k_t != "Name" && $k_t != "Modifier")){
                                
                                if(in_array($key_s, $slotdataColumns)){
                                    $index = array_search($key_s,$slotdataColumns);
                                    $slotdataValues[$index] = '"'.toDbString($this->mysqli, $na_v_c).'"';
                                }
                                else{
                                    $slotdataColumns[] = $key_s;
                                    $slotdataValues[] = '"'.toDbString($this->mysqli, $na_v_c).'"';
                                } 
                            }
                        }
                    }
                }
//                 $slotdataColumns = str_replace("%MORE_COLS%", $moreCols, $slotdataColumns);
//                 $slotdataValues = str_replace("%MORE_VALS%", $moreVals, $slotdataValues);
                if(isset($value['span_key'])){
                    $this->slotdata[$value['span_key']] =  getSlotData($this->mysqli, "(".join(',',$slotdataColumns).")", "(".stripslashes(join(',',$slotdataValues)).")");
                }
                
                $this->sql_array->add_element("slotdata", join(',',$slotdataColumns), join(',',$slotdataValues));
                
                if ($count_childern > 1){
                    foreach ($value['children'] as $child){
                        $child_slots = $this->qname2node[$section]["http://www.intuview.com/doc$this->dcid#".$child['key']] ?? null;
                        $slotdataColumns_child = ["docId", "conceptName", "conceptUrl"];
                        $slotdataValues_child = [$this->dcid, "\"$conceptName\"", "\"$child[key]\""];
                       
                        if(!empty($child_slots)){
                            foreach ($child_slots as $k_s_c => $v_s_c){
                                $e_c = explode("iv:" , $k_s_c);
                                if (count($e_c) > 1 && (array_key_exists($e_c[1],$slotToCaption))){
                                    $key_s_c = str_replace("-", "_", $e_c[1]);
                                    $k_t_c = $slotToCaption[$e_c[1]];
                                    $v_t_c = slotValueToCaption($v_s_c, $this->qname2node);
                                    if ($conceptName !== "person_object" || ($k_t_c != "Name" && $k_t_c != "Modifier")){
                                        $slotdataColumns_child[] = $key_s_c;
                                        $slotdataValues_child[] = '"'.toDbString($this->mysqli, $v_t_c).'"';
                                    }
                                }
                            }
                        }
                        if(!isset($this->slotdata[$child['span_key']])){
                            $this->slotdata[$child['span_key']] = getSlotData($this->mysqli, "(".join(',',$slotdataColumns_child).")", "(".stripslashes(join(',',$slotdataValues_child)).")");
                        }
                        
                        $this->sql_array->add_element("slotdata", join(',',$slotdataColumns_child), join(',',$slotdataValues_child));
                    }
                }
                
            }
        }
    }
    
    public function getCleanName($unescapedName, $organizationARC, $organizationARC_prefix){
        ///////////////////////
        // 1. Add company parameters (LTD, PLC, INC etc) to organization search
        // 2. Remove "The" "le" when searching organization
        // 3. If the organisation is composed of more than 3 tokens - then allow any with those three tokens
        // 4. If known organisation then return any organisation with that component
        ///////////////////////
        $e = explode(' ',$unescapedName);
        if(!isset($organizationARC_prefix)){
            $organizationARC_prefix = array();
        }
        if(!isset($organizationARC)){
            $organizationARC = array();
        }
        if (array_key_exists($e[0],$organizationARC_prefix)){
            $c = explode($e[0],$unescapedName);
            $name_c = trim($c[1]);
        }
        if (array_key_exists($e[sizeof($e) - 1],$organizationARC)){
            if (!empty($name_c) && strlen($name_c) > 0){
                $c = explode($e[0],$name_c);
                $e = explode($e[sizeof($e) - 1],$name_c);
                $name_c = trim($e[0]);
            }else{
                $e = explode($e[sizeof($e) - 1],$unescapedName);
                $name_c = trim($e[0]);
            }
        }
        if (empty($name_c)){
            $name_c = $unescapedName;
        }
        $name_c = toDbString($this->mysqli, $name_c);
        return $name_c;
        
    }
    
    
    public function colValsSlotdata($cols, $vals){
        $final_arr = array();
        $vals_arr = explode(',', $vals);
        $cols_arr = explode(',', $cols);
        
        for($i=0; $i < count($vals_arr); $i++){
            $e = trim(str_replace("_", "-", $cols_arr[$i]));
            $vals_arr[$i] = trim($vals_arr[$i]);
            $final_arr[$e] = trim($vals_arr[$i],'"');
            
        }
        return $final_arr;
        
    }
    
    public function insertTable2Values(){
        global $conceptTablesAsColumnstore;
        global $allConceptsTable;
        $generatedId = 1;
        foreach (array_keys($this->table2values) as $table)
        {
            foreach (array_keys($this->table2values[$table]) as $sqlColumns)
            {
                $columns = $sqlColumns;
                $firstInsertId = NULL;
                if ($conceptTablesAsColumnstore && $table != "slotdata")
                {
                    $firstInsertId = $generatedId;
                    //if tables are columnstore, id is not auto incremented, and should be generated manually
                    $columns = str_replace("(", "(id, ", $sqlColumns);
                    for ($index = 0; $index < count($this->table2values[$table][$sqlColumns]["values"]); $index++){
                        $this->table2values[$table][$sqlColumns]["values"][$index] =
                        str_replace("(", "(" . $generatedId++ . ", ", $this->table2values[$table][$sqlColumns]["values"][$index]);
                    }
                }
         
                $tableName = ($table == 'phrases') ? $table : $allConceptsTable; 
                
                $sql = "INSERT INTO $tableName $columns VALUES ";
                $sql .= join(", ", $this->table2values[$table][$sqlColumns]["values"]);
                write_to_log("TRACE", "insert entity $table");
                $res = sqlQuery($this->mysqli, $sql);
                if (!$res){
                    write_to_log("ERROR", "insert values failed");
                    continue;
                }
                if ($table == "slotdata"){
                    continue;
                }
                
                $insert_d = $this->mysqli->insert_id;
                if (!$firstInsertId){
                    $firstInsertId = $insert_d;
                }
                $this->table2values[$table][$sqlColumns]["keys"] = $this->table2values[$table][$sqlColumns]["keys"] ?? [];
                for ($insertIndex = 0; $insertIndex < count($this->table2values[$table][$sqlColumns]["keys"]); $insertIndex++)
                {
                    $insertId = $firstInsertId + $insertIndex;
                    $key = $this->table2values[$table][$sqlColumns]["keys"][$insertIndex];
                    $this->key2idConcept[$key]["id"] = $insertId;
                    if ($table == "person_object"){
                        $this->personKey2idName[$key]["id"] = $insertId;
                    }
                }
            }
        }	
      
    }
    
    public function addSubjectEntities(){
        global $subjectEntitiesBackground, $topicsTree;
        $topicsTreeReverse = array_flip($topicsTree);
        $intuview_subject =[];
        $intuview_subject_arr = [];
        
        $sql = "SELECT VALUE AS t FROM counted_theatre where docid=$this->dcid";
        $res = sqlQuery($this->mysqli, $sql);
        
        if(!empty($res)){
            while($obj = $res->fetch_object()){
                $val = trim($obj->t);
                if(!in_array($val, $intuview_subject)){
                    $intuview_subject_arr["intuview_subject_theatre"][] = trim($obj->t);    
                    $intuview_subject[] = trim($obj->t);
                }
            }
        }
        
        $sql = "SELECT VALUE AS t FROM document_topic where docid=$this->dcid";
        $res = sqlQuery($this->mysqli, $sql);
        
        if(!empty($res)){
            $topics = [];
            while($obj = $res->fetch_object()){
                $val = trim($obj->t);
       
                if(!in_array($val, $intuview_subject)){
                    $topics[] = $val;
                    $concept = $topicsTree[$val];
                    
                    $topics_tables[] = TablesInformation::getConceptPresentName($concept);
                }              
            }   
            //patch - remove father-son topics
            for($i = 0; $i < count($topics); $i++){  
                $concept = $topicsTree[$topics[$i]];
                $childTopics = array_values(Concept::getSubConcepts($this->mysqli, $concept));
                
                $intersect = array_intersect($topics_tables, $childTopics);
                if(!empty($intersect)){
                    unset($topics[$i]);
                }

            }
            $intuview_subject_arr["intuview_subject_topics"] = $topics;
            $intuview_subject  = array_merge($intuview_subject, $topics);
            
        }
      
        $concepts = [];
        $total_sum_concepts = 0;
        foreach($subjectEntitiesBackground["concepts"] as $table){
            if(!TablesInformation::isTableExists($this->mysqli, $table)){
               write_to_log("ERROR", "addSubjectEntities - table $table not exists - check config.php"); 
            }

            $sql = "SELECT SUM(occurrences) as occ FROM $table WHERE docid=$this->dcid";
            $res = sqlQuery($this->mysqli, $sql);
            
            
            if(!empty($res)){
                $obj = $res->fetch_object();
                if(!empty($obj->occ)){
                    $total_sum_concepts += $obj->occ;
                    $concepts[$table] = ["label"=>$table, "occ"=>$obj->occ];
                }
            }
        }
        if(!empty($concepts)){
            $avg_concepts = floor($total_sum_concepts/count($concepts));
            
            usort($concepts, function ($item1, $item2) {
                return $item2['occ'] <=> $item1['occ'];
            });
            foreach($concepts as $con){
                if($con['occ'] < $avg_concepts){
                    write_to_log("INFO", "Intuview-subject: removing $con[label], it got $con[occ], its under the avg: $avg_concepts");
                    continue;
                }
                $label = $topicsTreeReverse[$con['label']] ?? ucfirst(strtolower(str_replace("_"," ",$con["label"])));
                
                $intuview_subject_arr["intuview_subject_concepts"][] = $label;//"$label:$con[occ]";
                $intuview_subject[] = $label;
            }
        }
        
//         foreach($subjectEntitiesBackground["instances"] as $table){
//             if(!TablesInformation::isTableExists($this->mysqli, $table)){
//                 write_to_log("ERROR", "addSubjectEntities - table $table not exists - check config.php");
//             }
//             $sql = "SELECT NAME AS n FROM (
//             SELECT name FROM $table where docid=$this->dcid group by name order by occurrences DESC LIMIT 3) as t";
//             $res = sqlQuery($this->mysqli, $sql);
            
            
//             if(!empty($res)){
//                 while($obj = $res->fetch_object()){
//                     $val = trim($obj->n);
//                     if(!in_array($val, $intuview_subject)){
//                         $intuview_subject["instances"][] = trim($obj->n);
//                     }
//                 }
//             }
//         }
        
//         $sql = "SELECT subject s,relation_type t,object o FROM inter_entity_relationship WHERE docId=$this->dcid AND relation_type IN ('meeting', 'travel', 'communication')";
//         $res = sqlQuery($this->mysqli, $sql);
        
//         if(!empty($res)){
//             $topics = [];
//             while($obj = $res->fetch_object()){
//                 if(empty(trim($obj->t)) || empty($obj->o)){
//                     continue;
//                 }
//                 $val = "$obj->t between $obj->s and $obj->o";
                
                
//                 //check if we have sub-topics in the topics - if so skip
//                 if(!in_array($val, $intuview_subject)){
//                     $intuview_subject[] = $val;
//                 }
//             }
//         }
        
        if(!empty($intuview_subject)){
            $intuview_subject_str = implode("/", ($intuview_subject));
            $this->meta_arr['intuview_data']['intuview_subject_str'] = $intuview_subject_str;
            
            $this->meta_arr['intuview_data']['intuview_subject_obj'] = $intuview_subject_arr;        
          
        }        
        
        
    }
    
    public function insertTable2Values_old(){
        $curr_counter = 0;
        global $conceptTablesAsColumnstore;
        $generatedId = 1;
        foreach (array_keys($this->table2values) as $table)
        {

            foreach (array_keys($this->table2values[$table]) as $sqlColumns)
            {
                $columns = $sqlColumns;
                $firstInsertId = NULL;
                if ($conceptTablesAsColumnstore && $table != "slotdata")
                {
                    $firstInsertId = $generatedId;
                    //if tables are columnstore, id is not auto incremented, and should be generated manually
                    $columns = str_replace("(", "(id, ", $sqlColumns);
                    for ($index = 0; $index < count($this->table2values[$table][$sqlColumns]["values"]); $index++){
                        $this->table2values[$table][$sqlColumns]["values"][$index] =
                        str_replace("(", "(" . $generatedId++ . ", ", $this->table2values[$table][$sqlColumns]["values"][$index]);
                    }
                }
             
                
      
                if ($table == "slotdata"){
                    continue;
                }
                
                $values= join(", ", $this->table2values[$table][$sqlColumns]["values"]);
                $this->sql_array->add_element($table, $columns, $values);
              
                $this->table2values[$table][$sqlColumns]["keys"] = $this->table2values[$table][$sqlColumns]["keys"] ?? [];
                for ($insertIndex = 0; $insertIndex < count($this->table2values[$table][$sqlColumns]["keys"]); $insertIndex++)
                {
                    $key = $this->table2values[$table][$sqlColumns]["keys"][$insertIndex];
                    $this->key2idConcept[$key]["id"] = $curr_counter;
                    $curr_counter++;
                    if ($table == "person_object"){
                        $this->personKey2idName[$key]["id"] = $curr_counter;
                        $curr_counter++;
                    }
                }
            }
        }
        dumpDocsDbg("key2idConcept", $this->dcid, $this->key2idConcept);
        
    }
    
    public function sentConvertFunction(){
        global $allConceptsTable;
        $sentConvert = array("anti" => "anti_instance", "negative" => "anti_concept",
            "pro" => "pro_instance", "positive" => "pro_concept");
        if (!empty($this->sentimentArr)){
            foreach ($this->sentimentArr as $table => $v){
                foreach ($v as $name => $s){
                    $sql = "";
                    foreach ($s as $t => $value){
                        $concept_id = $this->getConceptId($t);
                        $sql = "SELECT id FROM $table WHERE docId=$this->dcid AND name='$name'";
                        $res = sqlQuery($this->mysqli, $sql);
                        if ($res!=FALSE){
                            $res = $res->fetch_object();
                            $instanceId = $res->id ?? null;
                            if (is_null($instanceId)){
                                $instanceId = 0;
                            }
                            foreach ($value as $vv){
                                $curr_val = toDbString($this->mysqli, $vv);
                                $this->pro_anti_array->add_element($sentConvert[$t], "docId,value,instanceId,instanceType", "$this->dcid,'$curr_val',$instanceId,'$table'");
                                if(in_array($t, array('pro','anti'))){
//                                     $this->pro_anti_array->add_element($t, "docId,name,username", "$this->dcid,'$curr_val','$this->username'");
                                    $this->pro_anti_array->add_element($allConceptsTable, "concept_type, concept_type_source, docId,name_version,username", "$concept_id, $concept_id, $this->dcid,'$curr_val','$this->username'");
                                    
                                }
//                                 if(array_key_exists(TablesInformation::getConceptPresentName($table), $this->subClass2classes)){
//                                     foreach($this->subClass2classes[TablesInformation::getConceptPresentName($table)] as $parent_table){
//                                         $parent_table = TablesInformation::getTableName($parent_table);
//                                         if(!array_key_exists(strtolower($parent_table),$this->merged_data)){
//                                             continue;
//                                         }
//                                         $parent_table_id = $this->getConceptId($parent_table);
//                                         $this->pro_anti_array->add_element($allConceptsTable, "concept_type, concept_type_source, docId,name_version", "$parent_table_id,$this->dcid,'$curr_val'");    
//                                     }
//                                 }
                                
                            }
                          
                        }
                    }
                }
            }
        }
    }
    
    public function updateProAntiAgg(){
        //update aggregated pro/anti
        if (array_key_exists("anti", $this->aggregated_sentiment)){
            sort($this->aggregated_sentiment["anti"]);
        }
        
        if (array_key_exists("pro", $this->aggregated_sentiment)){
            sort($this->aggregated_sentiment["pro"]);
        }
        $aggregated_anti= toDbString($this->mysqli, isset($this->aggregated_sentiment["anti"])) ? toDbString($this->mysqli, implode(";", array_unique($this->aggregated_sentiment["anti"]))) : "";
        $aggregated_pro = toDbString($this->mysqli, isset($this->aggregated_sentiment["pro"])) ? toDbString($this->mysqli, implode(";", array_unique($this->aggregated_sentiment["pro"]))) : "";
        $query = "UPDATE files
    	SET anti=\"$aggregated_anti\", pro=\"$aggregated_pro\"
    	WHERE id=$this->dcid";
        //is it needed????
        sqlQuery($this->mysqli,$query);
        
    }
    

    public function relationHandler(){
        global $excluded_relations;
        write_to_log("TRACE", $this->dcid . " Background: inserting relations to DB");
        $final_array = array();
        //Inter-entity-relationship
        $sql = "SELECT * FROM inter_entity_relationship WHERE docId=$this->dcid";
        $res = sqlQuery($this->mysqli,$sql);
        if (!empty($res)) {
            $row = $res->fetch_object();
            if (empty($row)){ //only if empty we will continue here
                
                
                //create array of lowercase caption => table
                $arrCaptionTable = array();
                foreach ($this->pred2inst["dig"]["iv:caption"] as $k => $v){
                    if (array_key_exists("iv:entity-type", $v[0])){
                        $arrCaptionTable[strtolower($k)] = unqualifyName($v[0]["iv:entity-type"]);
                    }
                }
                foreach ($this->key2idConcept as $k => $v){
                    $e = array();
                    if (strpos($k,"dynamic-inst-rt") !== false){
                        $e = explode("dynamic-inst-rt",$k);
                    }
                    elseif (strpos($k,"dynamic-inst-ref") !== false){
                        $e = explode("dynamic-inst-ref",$k);
                    }
                    if (sizeof($e) > 0)
                    {
                        $e = explode("-1",$e[1]);
                        $name = substr($e[0],1);
                        if(isset($v["table"])){
                            $arrCaptionTable[strtolower($name)] = $v["table"];
                        }
                    }
                }
                
                
                
                $json = $this->jsonRelationCreate();
   
                
                $obj_arr = array();
                $sub_arr = array();
                
                if (isset($json['results']['bindings']) && $json['results']['bindings']){
                    
                    /**
                     * add to digestJson the relationship element
                     */
                    //create array of digestcaption & flavors
                    $settings_class =  new Settings($this->mysqli, $this->username);
                    $flavors_arr =  $settings_class->getSettings("be", "getFlavors");
                    foreach($flavors_arr as $item){
                        $digestCaption[] = "Relationship";
                        $flavors[] = $item["name"];
                    }    
                    $digestJsonRelation = array("title" => "Inter-Entity-Relationship",
                        "digestCaption" => $digestCaption,
                        "flavor" => $flavors,
                        "key" => "inter-entity-relationship",
                        "isFolder" => true,
                        "children" => [],
                        "type" => "http://www.intuview.com/ontology#Inter-entity-relationship"
                    );
                    
                    foreach ($json['results']['bindings'] as $value){
                        if (($value['subj']['xml:lang'] == 'en' && $value['obj']['xml:lang'] == 'en') ||
                            !key_exists("subj", $value)){
                                $xkey = unqualifyName($value['relId']['value']);
                                $elements = $this->getSpanElements($value['relId']['value']);
                                if (!is_null($elements)) {
                                    $y = 0;
                                    foreach ($elements as $delement) {
                                        $arr_rel[$delement->getAttribute("start")] = array($delement->getAttribute("end"), $xkey.$y++);
                                    }
                                }
                                $xkey = unqualifyName($value['se']['value']);
                                $elements = $this->getSpanElements($value['se']['value']);
                                if (!is_null($elements)) {
                                    $y = 0;
                                    foreach ($elements as $delement) {
                                        $arr_se[$delement->getAttribute("start")] = array($delement->getAttribute("end"), $xkey.$y++);
                                    }
                                }
                                $xkey = unqualifyName($value['oe']['value']);
                                $elements = $this->getSpanElements($value['oe']['value']);
                                if (!is_null($elements)) {
                                    $y = 0;
                                    foreach ($elements as $delement) {
                                        $arr_oe[$delement->getAttribute("start")] = array($delement->getAttribute("end"), $xkey.$y++);
                                    }
                                }
                                $object_url = unqualifyName($value['oe']['value']) . "_0";
                                $obj_arr[$object_url] = $value['obj']['value'];
                                $object = $value['obj']['value'];
                                $subject = $value['subj']['value'];
                                $entity2idTable = array();
                                foreach (array("subj" => $value['se']['value'], "obj" => $value['oe']['value']) as $type => $entity)
                                {
                                    if (array_key_exists(unqualifyName($entity), $this->key2idConcept))
                                    {
                                        $entity2idTable[$type] = $this->key2idConcept[unqualifyName($entity)];
                                        continue;
                                    }
                                    
                                    if (array_key_exists($entity, $this->pred2inst["agg"]["iv:aggregated-from"] ?? [])){
                                        $entity = $this->pred2inst["agg"]["iv:aggregated-from"][$entity][0]["rdf:about"];
                                    }
                                    elseif (!empty($caption2aggemail) && array_key_exists($value[$type]["value"], $caption2aggemail)){
                                        $entity = $caption2aggemail[$value[$type]["value"]]["rdf:about"];
                                    }
                                    
                                    $entity = unqualifyName($entity);
                                    if (array_key_exists($entity, $this->personKey2idName))
                                    {
                                        $entity2idTable[$type]["id"] = array_key_exists('id',$this->personKey2idName[$entity]) ?
                                        $this->personKey2idName[$entity]["id"] : 0;
                                        $entity2idTable[$type]["table"] = "person_object";
                                    }
                                    else if (array_key_exists($entity, $this->key2idConcept))
                                    {
                                        $entity2idTable[$type] = $this->key2idConcept[$entity];
                                    }
                                    else {
                                        $ont_name = "";
                                        $e = array();
                                        if (strpos($entity,"dynamic-inst-rt") !== false){
                                            $e = explode("dynamic-inst-rt",$entity);
                                        }
                                        elseif (strpos($entity,"dynamic-inst-ref") !== false){
                                            $e = explode("dynamic-inst-ref",$entity);
                                        }
                                        if (sizeof($e) > 0)
                                        {
                                            $e = explode("-1",$e[1]);
                                            $ont_name = substr($e[0],1);
                                        }else{
                                            $ont_name = $entity;
                                        }
                                        
                                        
                                        if (array_key_exists($ont_name, $arrCaptionTable)){
                                            $entity2idTable[$type] = array("id" => 0, "table" => $arrCaptionTable[$ont_name]);
                                        }
                                        elseif (array_key_exists(strtolower($ont_name), $arrCaptionTable)){
                                            $entity2idTable[$type] = array("id" => 0, "table" => $arrCaptionTable[strtolower($ont_name)]);
                                        }elseif (!empty($ont_name) && (strpos($ont_name,"province") !== false || strpos($ont_name,"city") !== false || strpos($ont_name,"cities") !== false|| strpos($ont_name,"national_identity") !== false)){
                                            $entity2idTable[$type] = array("id" => 0, "table" => "place_object");
                                        }
                                        else{
                                            if ($type == "obj"){
                                                $ont_name = $object;
                                            }
                                            else{
                                                $ont_name = $subject;
                                            }
                                            if (array_key_exists($ont_name, $arrCaptionTable)){
                                                $entity2idTable[$type] = array("id" => 0, "table" => $arrCaptionTable[$ont_name]);
                                            }
                                            elseif (!empty($ont_name) && (array_key_exists(strtolower($ont_name), $arrCaptionTable))){
                                                $entity2idTable[$type] = array("id" => 0, "table" => $arrCaptionTable[strtolower($ont_name)]);
                                            }elseif (!empty($ont_name) && (strpos($ont_name,"province") !== false || strpos($ont_name,"city") !== false || strpos($ont_name,"cities") !== false|| strpos($ont_name,"national_identity") !== false)){
                                                $entity2idTable[$type] = array("id" => 0, "table" => "place_object");
                                            }
                                            else{
                                                write_to_log("WARNING", "$type key not found in table: $entity");
                                                $entity2idTable[$type] = array("id" => 0, "table" => "");
                                            }
                                        }
                                    }
                                    $entity2idTable[$type]["table"] = str_replace("-","_",$entity2idTable[$type]["table"]);
                                    if (!empty($ont_name) && (strpos($entity2idTable[$type]["table"],"province") !== false || strpos($entity2idTable[$type]["table"],"city") !== false || strpos($entity2idTable[$type]["table"],"cities") !== false|| strpos($entity2idTable[$type]["table"],"national_identity") !== false)){
                                        $entity2idTable[$type]["table"] = "place_object";
                                    }
                                    if (!empty($ont_name) && (strpos($entity2idTable[$type]["table"],"military_organization") !== false)){
                                        $entity2idTable[$type]["table"] = "organizational_identity";
                                    }
                                }
                                
                                $rel_type = $value['rel_type'];//$this->get_relationship_type($value["rel_type"]["value"]);
                                $rel_type_inst = $value['rel_type_inst'];
                                
                                $object_url = toDbString($this->mysqli, $object_url);
                                $object = toDbString($this->mysqli, $object);
                                $subject_url = unqualifyName($value['se']['value']) . "_0";
                                $sub_arr[$subject_url] = $value['subj']['value'];
                                $subject_url = toDbString($this->mysqli, $subject_url);
                                $subject = toDbString($this->mysqli, $subject);
                                $rel_url = unqualifyName($value['relId']['value']) . "0";
                                $rel_arr[$rel_url] = $value['relId']['value'];
                                $rel = $value['relId']['value'];
                                $rel = toDbString($this->mysqli, $rel);
                                $rel = unqualifyName($rel);
                                $subjectTable = $entity2idTable['subj']['table'];
                                  
                                $span_key = self::get_key_forNoIndex($value['se']['value']) ?? null;
                              
                                $digestJsonRelation["children"][] = 
                                array( "title" => "$subject => $rel_type => $object",
                                    "key" => $span_key ?? null,
                                        "ontUrl" => "",
                                        "type" => "http://www.intuview.com/ontology#Inter-entity-relationship"
                                    
                                );
                                //add the relationship title to key_data
                                if(!array_key_exists("relationship", $this->metaParameter)){
                                    $this->metaParameter["relationship"] = "";
                                }
                                $this->metaParameter["relationship"] .= "$subject => $rel_type => $object; "; 
                                
                                if (empty($subjectTable)){
                                    if (strpos($subject_url, "cities") !== false || strpos($subject_url, "population-centers") !== false
                                        || strpos($subject_url, "provinces"))
                                        $subjectTable = "place_object";
                                        elseif (strpos($subject_url, "Organizational-identity") !== false)
                                        $subjectTable = "organizational_identity";
                                }
                                $objectTable = $entity2idTable['obj']['table'];
                                if (empty($objectTable)){
                                    if (strpos($object_url, "cities") !== false || strpos($object_url, "population-centers") !== false
                                        || strpos($object_url, "provinces")){
                                        $objectTable = "place_object";
                                    }
                                    elseif (strpos($object_url, "Organizational-identity") !== false){
                                        $objectTable = "organizational_identity";
                                    }
                                }
                                $objectTable = strtolower($objectTable);
                                $subjectTable = strtolower($subjectTable);
                                $subjectIndex = array_key_exists("id", $entity2idTable["subj"])? $entity2idTable["subj"]["id"] : 0;
                                $objectIndex = array_key_exists("id", $entity2idTable["obj"])? $entity2idTable["obj"]["id"] : 0;
                                
                                //special care for electronic emails
                                if($rel_type_inst=='electronic-mail'){
                                    write_to_log("TRACE", $this->dcid . ": relations email - adding to slotdata");
                                    $this->handleEmailsRelations($subject, $subject_url, $object);
                                }
                                
                                //skip excluded relations type
                                if(in_array($rel_type_inst, $excluded_relations)){
                                    write_to_log("TRACE", $this->dcid . ": relations $rel_type in excluded types");
                                    continue;
                                }
                                
                                
                                $values = "$this->dcid,'$rel','$rel_url','$subject','$subject_url','$subjectIndex','$subjectTable','$rel_type',\"$rel_type_inst\",'$object','$object_url','$objectIndex','$objectTable'";
                                if($_REQUEST['getmetadata']){
                                    if(empty($this->meta_arr['intuview_data']['relationships'])){
                                        $this->meta_arr['intuview_data']['relationships'] = [];
                                    }
                                    if(!in_array("$subject ==> $rel_type ==> $object", $this->meta_arr['intuview_data']['relationships'])){
                                         $this->meta_arr['intuview_data']['relationships'][] = "$subject ==> $rel_type ==> $object";
                                    }
                                }
                                $keys = "docId,relationship,relationship_url,subject,subject_url, subject_id, subject_table, relation_type,relation_type_inst,object,object_url, object_id, object_table";
                                $this->sql_array->add_element("inter_entity_relationship", $keys, $values);       
                        }
                    }
                }
                
                $arr_rel_word = array();
                if (!empty($arr_rel)){
                    foreach ($arr_rel as $letterIndex => $endAndKey)
                    {
                        $arr_rel_word[$endAndKey[1]] = mb_substr($this->text, $letterIndex, $endAndKey[0] + 1 - $letterIndex, 'UTF-8');
                    }
                }
                
                $arr_obj_word = array();
                if (!empty($arr_oe)){
                    foreach ($arr_oe as $letterIndex => $endAndKey)
                    {
                        $arr_obj_word[$endAndKey[1]] = mb_substr($this->text, $letterIndex, $endAndKey[0] + 1 - $letterIndex, 'UTF-8');
                    }
                }
                
                $arr_sub_word = array();
                if (!empty($arr_se)){
                    foreach ($arr_se as $letterIndex => $endAndKey)
                    {
                        $arr_sub_word[$endAndKey[1]] = mb_substr($this->text, $letterIndex, $endAndKey[0] + 1 - $letterIndex, 'UTF-8');
                    }
                }
            }
        }
        $final_array['obj_arr'] = $obj_arr  ?? null;
        $final_array['sub_arr'] = $sub_arr  ?? null;
        $final_array['rel_arr']= $rel_arr ?? null;
        
        $final_array['arr_obj_word'] = $arr_obj_word  ?? null;
        $final_array['arr_sub_word'] = $arr_sub_word ?? null;
        $final_array['arr_rel_word'] = $arr_rel_word  ?? null;
        
        //add the digestJsonRelationship element to digestJson
        if(!empty($digestJsonRelation['children'])){
               array_unshift($this->digestJson, $digestJsonRelation);
        }
        return $final_array;
    }
    
    public function handleEmailsRelations($name, $subject_url, $email){
        $subject_url = rtrim($subject_url,'_0');
       //first update slotdata info with this email of the person
       $sql = "UPDATE slotdata SET email='$email'   WHERE conceptUrl='$subject_url' AND docId=$this->dcid";
       $this->emails_sql[] = $sql;
    }
    
    public function handleEmailsSql(){
        foreach($this->emails_sql as $sql){
            sqlQuery($this->mysqli,$sql);  
        }
    }
    
    
    public function updateBackgroundRelations($rel_arr, $obj_arr, $sub_arr, $arr_obj_word, $arr_sub_word, $arr_rel_word){
        write_to_log("TRACE", $this->dcid . " Background: start inserting relations to DB");
        if(isset($obj_arr)){
            foreach ($obj_arr as $k=>$v){
                if (key_exists($k, $arr_obj_word)){
                    $o = toDbString($this->mysqli, $arr_obj_word[$k]);
                    $k = toDbString($this->mysqli, $k);
                    $sql = "UPDATE inter_entity_relationship
        					SET object_text='$o'
        					WHERE object_url='$k' AND docId=$this->dcid";
                    sqlQuery($this->mysqli,$sql);
                }
            }
        }
        
        if(isset($sub_arr)){
            foreach ($sub_arr as $k=>$v){
                if (key_exists($k, $arr_sub_word)){
                    $o = toDbString($this->mysqli, $arr_sub_word[$k]);
                    $k = toDbString($this->mysqli, $k);
                    $sql = "UPDATE inter_entity_relationship
        					SET subject_text='$o'
        					WHERE subject_url='$k' AND docId=$this->dcid";
                    sqlQuery($this->mysqli,$sql);
                }
            }
        }
        
        if (isset($rel_arr) && $rel_arr){
            foreach ($rel_arr as $k=>$v){
                if (key_exists($k, $arr_rel_word)){
                    $o = toDbString($this->mysqli, $arr_rel_word[$k]);
                    $k = toDbString($this->mysqli, $k);
                    $sql = "UPDATE inter_entity_relationship
        					SET relationship_text='$o'
        					WHERE relationship_url='$k' AND docId=$this->dcid";
                    sqlQuery($this->mysqli,$sql);
                }
            }
        }
        write_to_log("TRACE", $this->dcid . " Background: finished inserting relations to DB");
    }
    
    
    public function addMenuColors(){
        global $nonAggregatedTreatedAsAggregated;
        write_to_log("TRACE", $this->dcid . " Background: menu colors");
        for ($d = 0; $d < sizeof($this->digestJson); $d++){
            for ($c = 0; $c < sizeof($this->digestJson[$d]['children']); $c++){
                $title = $this->digestJson[$d]['children'][$c]['title'];
                if(in_array($title, $nonAggregatedTreatedAsAggregated)){
                    continue;
                }
                if(is_array($title)){
                    $title = reset($title);
                }
                if (key_exists("children", $this->digestJson[$d]['children'][$c])){
                    for ($k = 0; $k < sizeof($this->digestJson[$d]['children'][$c]['children']); $k++){
                        $title_item = $this->digestJson[$d]['children'][$c]['children'][$k]['title'];
                        if(is_array($title_item)){
                            $title_item = reset($title_item);
                        }
                        if (strcasecmp($title,$title_item)){
                            $t = ($title_item) ?? null;
                            $j = $k+1;
                            $jTitle = $this->digestJson[$d]['children'][$c]['children'][$j]['title']??null;
                            if(is_array($jTitle)){
                                $jTitle = $jTitle[0];
                            }
                            $cChildren = $this->digestJson[$d]['children'][$c]['children'] ?? null;
                            while ($t == $jTitle && $j < sizeof($cChildren)){
                                $j++;
                            }
                            
                            if (($j-1) > $k){
                                $arr = array();
                                $arr['icon'] = false;
                                $arr['title'] = $this->digestJson[$d]['children'][$c]['children'][$k]['title'];
                                $arr['key'] = $this->digestJson[$d]['children'][$c]['children'][$k]['title'];
                                $arr['children'] = array();
                                for ($i = $k, $t = 0; $i < $j; $i++,$t++){
                                    $arr['children'][$t] = $this->digestJson[$d]['children'][$c]['children'][$i];
                                }
                                for ($i = $k; $i < $j; $i++){
                                    unset($this->digestJson[$d]['children'][$c]['children'][$i]);
                                }
                                $this->digestJson[$d]['children'][$c]['children'][$k] = $arr;
                                
                            }
                        }
                    }
                }
            }
        }
        
    }
    

    public function addEmotions(){  
        
        //1 - All instances in the text
//         $total_concept = 0;
//         foreach($this->table2values['concept'] as $element){
//             $total_concept += count($element['values']);
//         }
        $total_concept = $this->total_concepts;
        //2 - all Positive-object-indicator & Anti-object-indicator instances 
        $pro = self::count_sentiment('pro');      
        $anti = self::count_sentiment('anti');     
        
        //3 - all Sentiment-indicator-type (Pos+anti)
        $total_sent = ($pro + $anti) ;
        
        //4 - All Sentiment-indicator-type divided by all Concepts
        $emotion =  $total_concept==0 || $total_sent == 0 ? 0 : number_format(($total_sent / $total_concept) * 100, 2);
        
        //5 - All Positive divided by all concepts = % of concepts that are positive 
        $pro_precent = $total_concept==0 || $total_sent == 0 ? 0 :  number_format(($pro / $total_sent) * 100, 2);
        
        //6 -  All negative divided by all concepts = % of concepts that are negative
        $anti_percent = $total_concept==0 || $total_sent == 0 ? 0 :  number_format(($anti / $total_sent) * 100, 2);
        
        //7 - Positive minus negeative = mood
        $mood = number_format(($pro_precent - $anti_percent),2) ;
        $this->emotion_arr = array('pro' => $pro_precent, 'anti' => $anti_percent, 
            'emotion' => $emotion, 'mood' => $mood);
 
		$emotion = number_format((float)$emotion, 2, '.', '');
		
		if($_REQUEST['getmetadata']){
		    $this->meta_arr["intuview_data"]["emotion_arr"] = $this->emotion_arr;
		}
        $this->sql_array->add_element("emotion", "`docid`,`anti`,`pro`,`e_concept`", "$this->dcid,'$anti_percent','$pro_precent','$emotion'");        
    }
    
    public function count_sentiment($sent){
//         $sent_value = 0;
//         if(!empty($this->pro_anti_array->insert_array[$sent])){
//             foreach($this->pro_anti_array->insert_array[$sent] as $element){
//                 $sent_value += count($element);
//             }
//         }
//         if($sent_value == 0){
//             $sql = "SELECT COUNT(*) AS c FROM $sent WHERE $sent.docid=$this->dcid";
//             $res = sqlQuery($this->mysqli,$sql);
//             if($res!==false){
//                 $row = $res->fetch_object();
//                 $sent_value = intval($row->c);
//             }
//         }
        $sent_value = 0;
        foreach($this->digestJson as $element){
            $parents_arr = [];
            $type = str_replace("http://www.intuview.com/ontology#", "",$element["type"]);
            
            if(is_string($element['type'])){
                $parents_arr = Concept::getParentConcepts($this->mysqli, $type);
            }
            else{
                foreach ($element['type'] as $t){
                    $parents_arr = array_merge($parents_arr, Concept::getParentConcepts($this->mysqli, $t));
                }
            }
            if((($type == 'Positive-object-indicator' || in_array('Positive-object-indicator', $parents_arr)) 
                && $sent == 'pro') || 
              (($type == 'Anti-object-indicator' || in_array('Anti-object-indicator', $parents_arr))
                    &&  $sent == 'anti')){
                $sent_value = count($element['children']);
                break;
            }    
        }
        return $sent_value;
    }
    
    
    public function storeResultFile($srcfile, $shared = false)
    {
        $path_parts = pathinfo($srcfile);
        
        if($shared){
            global $sharedFolder;
            $desfile = $sharedFolder;
        }
        else{
            $desfile = $path_parts['dirname'];
            $desfile.= DIRECTORY_SEPARATOR . "..";          
        }
        $desfile.= DIRECTORY_SEPARATOR . "done_results";
        if( !is_dir($desfile) ) {
            mkdir($desfile);
        }
        $desfile.= DIRECTORY_SEPARATOR . $path_parts['basename'];
        rename($srcfile, $desfile); //mark file as done
    }
    
    public function saveMetaDataToZip($directory, $textDir, $report, $menuColorStyle){
        //MetaDate
        write_to_log("TRACE", $this->dcid . " Background: saving metadata files");
        
        $zip_arr = [];
        $zip_arr["m_$this->dcid.html.txt"] = $menuColorStyle;
        usort($this->digestJson,  function ($item1, $item2) {
            return $item1['title'] <=> $item2['title'];
        });
        $zip_arr["d_$this->dcid.html.txt"] = json_encode($this->digestJson);
        $zip_arr["n_$this->dcid.html.txt"] = $this->newText;
        $zip_arr["t_$this->dcid.html.txt"] = $textDir;
        $zip_arr["r_$this->dcid.html.txt"] = $report;
        $zip_arr["s_$this->dcid.html.txt"] = json_encode($this->slotdata ?? null);
        $zip_arr["e_$this->dcid.html.txt"] = json_encode($this->emotion_arr ?? null);
        
        
        global $saveRDF;
        if ($saveRDF)
        {
            $zip_arr["responseRDF.html"] = $this->blob;
            $zip_arr["text.html.txt"] = $this->text;
        }
        
        //iterates and creates different zip for each data type
        foreach($zip_arr as $key=>$zip_element) {     
                // Name of the gz file we're creating
                $gzfile = $directory.DIRECTORY_SEPARATOR."Background_".substr($key, 0 , (strrpos($key, "."))).".gz"; 
                $gzopenRetryCount = 0;
                // Open the gz file (w9 is the highest compression)
                $fp = gzopen ($gzfile, 'wb9');
				while ($fp === FALSE)
				{
					if (++$gzopenRetryCount > 10) {
						break;
					}
					
					write_to_log("ERROR", "Failed creating gzip. Sleeping and retrying. count: " . $gzopenRetryCount);
					sleep(2);
					$fp = gzopen ($gzfile, 'wb9');
				}                
                // Compress the file
                $fp && gzwrite ($fp, $zip_element);
                
                // Close the gz file and we're done
                $fp && gzclose($fp);   
        }
    
    }

    public function saveMetaDataToDB($textDir, $report, $menuColorStyle){
        $res = sqlQuery($this->mysqli, "SHOW TABLES LIKE 'metadata'");
        if($res == false){
               write_to_log("TRACE", $this->dcid . " no metadata table is configured yet");
            
        }
        //MetaDate
        write_to_log("TRACE", $this->dcid . " Background: saving metadata to DB");
        
        $this->meta_arr["menuColorStyle"] = $menuColorStyle;
        usort($this->digestJson,  function ($item1, $item2) {
            return $item1['title'] <=> $item2['title'];
        });
            $this->meta_arr["digestJson"] = json_encode($this->digestJson);
            $this->meta_arr["newText"] = $this->newText;
            $this->meta_arr["textDir"] = $textDir;
            $this->meta_arr["report"] = $report;
            $this->meta_arr["slotdata"] = json_encode($this->slotdata ?? null);
            $this->meta_arr["emotion_arr"] = json_encode($this->emotion_arr ?? null);
            $this->meta_arr["key_data"] = json_encode($this->filter_key_data_params($this->metaParameter ?? null));
            
            global $saveRDF;
            if ($saveRDF)
            {
                $this->meta_arr["responseRDF.html"] = $this->blob;
                $this->meta_arr["text.html.txt"] = $this->text;
            }
            $meta_json = json_encode($this->meta_arr, true);
            
            $meta_comress = base64_encode(gzcompress($meta_json,9));
            $meta_compress_sql = $this->mysqli->real_escape_string($meta_comress);
            $sql = "INSERT INTO metadata (docId, metadata) VALUE($this->dcid, '$meta_compress_sql')";
            
            $res = sqlQuery($this->mysqli, $sql);
            
    }
    
    public function filter_key_data_params($arr){
        $final = [];

//         $query ="SELECT ui_settings->\"$.key_data\" AS settings FROM user_settings WHERE username ='". $this->username ."'";
        $query = "SELECT JSON_UNQUOTE(JSON_EXTRACT(ui_settings, '$.key_data')) AS settings
            FROM user_settings
            WHERE  username ='". $this->username ."'";
        $res = sqlQuery($this->mysqli, $query);
        $obj = $res->fetch_object();
        if(!empty($obj)){
            $allowed = json_decode($obj->settings, true);
            foreach($arr as $key=>$val){
                $index = array_search($key, array_column($allowed, 'field'));
                if($index === false){
                    continue;
                }
                $title = $allowed[$index]["title"];
                $caption = $allowed[$index]["caption"];
                
                $final[$title][$caption] = $val;       
            }  
        }
        return $final;
    }
    
    public function insertNameAlt(){
        write_to_log("TRACE", $this->dcid . " Background: name alternatives");
        
        $name2additionalInfo = populateNameAlternatives($this->personKey2idName, $this->pred2inst, $this->qname2node);
        //insert name additional info to DB
        insertNaAdditionalInfoToDB($name2additionalInfo, $this->mysqli);
        
        $watchListName = "DOCEX Entities DB";
        $this->username = toDbString($this->mysqli, $this->username);
        
        
        //insert all NAs to DB
        insertNAsToDB($this->mysqli, $this->dcid, $this->personKey2idName, "person_object", $watchListName, $this->username);
        dumpDocsDbg("background_personKey2idName", $this->dcid, $this->personKey2idName);
        
    }
    
    public function insertEmails(){
        //insert email NAs
        $emailKey2idName = array();
        $watchListName = "DOCEX Entities DB";
        foreach ($this->key2idConcept as $key => $idTable)
        {
            if (!isset($idTable["table"])  || $idTable["table"] != "email"){
                continue;
            }
            
            $emailKey2idName[$key]["id"] = $idTable["id"];
            $emailCaption = $this->qname2node["dig"][qualifyName($key, $this->metaParameter["terminal_system_id"])]["iv:caption"];
            if (is_array($emailCaption)){
                $emailCaption = $emailCaption["en"];
            }
            $emailCaption = explode("@", $emailCaption);
            $emailCaption = $emailCaption[0];
            $emailCaption = str_replace(array(".", "-", "_"), " ", $emailCaption);
            $emailKey2idName[$key]["name"] = $emailCaption;
        }
        $name2additionalInfo = populateNameAlternatives($emailKey2idName, $this->pred2inst, $this->qname2node, true);
        dumpDocsDbg("emailKey2idName", $this->dcid, $emailKey2idName);
        //insert name additional info to DB
        insertNaAdditionalInfoToDB($name2additionalInfo, $this->mysqli);
        //insert all NAs to DB
        insertNAsToDB($this->mysqli, $this->dcid, $emailKey2idName, "email", $watchListName, $this->username);
        
        
    }
    
    /**
     * guessCaptionFromIvname
     * @param  $inst
     * @return $xcap
     */
    public function guessCaptionFromIvname($inst) {
        $xcap = "";
        
        $ontUrl = array_key_exists("iv:instance",$inst)?$inst["iv:instance"]:"";        
        if(!empty($ontUrl)){
            $tInst_alt = $this->pred2inst["dig"]["iv:instance"][$ontUrl][0];
            if(!empty($tInst_alt) && array_key_exists("iv:caption", $inst)){

                $xcap =  $tInst_alt["iv:transliteration-ic"] ?? $tInst_alt["iv:caption"] ?? null;
            }
        }
        if (empty($xcap) && !empty($inst["iv:transliteration-ic"])){
            if(is_array($inst["iv:transliteration-ic"]) && array_key_exists("en", $inst["iv:transliteration-ic"])){
                $inst["iv:transliteration-ic"] = $inst["iv:transliteration-ic"]["en"];
            }

            if(strstr($inst["iv:transliteration-ic"], "#extern")){
                $ICArr = $this->getExternToArr($inst["iv:transliteration-ic"]);
                $inst["iv:transliteration-ic"] = $ICArr["English"] ?? $inst["iv:transliteration-ic"];
            }
            $xcap = $inst["iv:transliteration-ic"];
        }
        elseif (empty($xcap) && array_key_exists("iv:caption", $inst)){
            $xcap =  $inst["iv:caption"];
        }
        elseif (empty($xcap) && array_key_exists("iv:translated-name", $inst)){
            $xcap = $inst["iv:translated-name"];
        }
        elseif (empty($xcap) && array_key_exists("iv:name", $inst)) {
            $xcap = $inst["iv:name"];
        }elseif(empty($xcap) && array_key_exists("iv:phone-number", $inst)){
            $xcap = "Telephone-number";//$inst["iv:phone-number"];
        }
        
        if (is_array($xcap) && array_key_exists("en", $xcap)) {
            $xcap = $xcap["en"];
        }
        
        if(is_array($xcap)){
            $xcap = $xcap[0] ?? null;
        }
        if(!empty($xcap) && strstr($xcap, "#extern")){
            $ICArr = $this->getExternToArr($xcap);
            $xcap = $ICArr["English"] ?? xcap;
            
        }
        return $xcap;
    }
    
    function getExternToArr($string){
        $result = [];
        
        if (preg_match('/\[([^]]+)\]/', $string, $matches)) {
            $content = $matches[1];
            
            // Split the content by commas
            $pairs = explode(',', $content);
            
            
            // Iterate through key-value pairs
            foreach ($pairs as $pair) {
                // Split each pair by colon
                list($key, $value) = explode(':', $pair, 2);
                
                // Remove leading and trailing whitespaces
                $key = trim($key);
                $value = trim($value);
                
                // Add key-value pair to the result array
                $result[$key] = $value;
            }
        }
        return $result;
    }
    
    /**
     * getDigestSpanObjectNames
     * @param  $instanceReferences
     * @return $objectNames
     */
    public function getDigestSpanObjectNames($instanceReferences)
    { //currently only relevant to Aya in pred2inst["dig"]["iv:name"]
        //but I keep it here for reference
        $objectNames = array();
        foreach ($instanceReferences as $ref) {
            if( array_key_exists("iv:aggregated-from", $ref) ) {
                if( is_array($ref["iv:aggregated-from"]) ) {
                    $objectNames = array_merge($objectNames, $ref["iv:aggregated-from"]);
                }
                else {
                    $objectNames[] = $ref["iv:aggregated-from"];
                }
            } else {
                $objectNames[] = $ref["rdf:about"];
            }
        }
        return $objectNames;
    }
    
    /**
     * checkIfExistsInDigest
     * @param  $about
     * @return $exists - boolean
     */
    public function checkIfExistsInDigest($about)
    {
        $exists = array_key_exists($about, $this->checkIfExistsInDigest_hash);
        $this->checkIfExistsInDigest_hash[$about] = true;
        return $exists;
    }
    
    /**
     * aggregateFromPredicates
     * @param  $pred2instName
     * @return $caption2agg - array values of it
     */
    public function aggregateFromPredicates($pred2instName)
    {
        $caption2agg = array();
        foreach ($pred2instName as $item)
        {
            $caption = $item["iv:caption"] ?? null;
            if (is_array($caption)){
                $caption = $caption["en"];
            }
            
            if (!array_key_exists($caption, $caption2agg))
            {
                $item["iv:aggregated-from"] = $item["rdf:about"];
                $caption2agg[$caption] = $item;
//                 $caption2agg[$caption] = array(
//                     "rdf:about" => $item["rdf:about"],
//                     "iv:caption" => array("en" => $caption),
//                     "rdf:type" => $item["rdf:type"],
//                     "iv:aggregated-from" => $item["rdf:about"]
//                 );
            }
            else
            {
                $aggFrom = $caption2agg[$caption]["iv:aggregated-from"];
                if (is_array($aggFrom)){
                    $caption2agg[$caption]["iv:aggregated-from"][] = $item["rdf:about"];
                }
                else{
                    $caption2agg[$caption]["iv:aggregated-from"] = array($aggFrom, $item["rdf:about"]);
                }
                foreach($item as $slot=>$value){
                    if(!array_key_exists($slot, $caption2agg[$caption])){
                        $caption2agg[$caption][$slot] = $value;
                    }
                }
            }
        }
        return array_values($caption2agg);
    }
    
    
    function getSpanElements($objectName)
    {
        global $huge_files;
        $result = [];
        if($huge_files){
            set_time_limit(0); //special flag for huge files as 200+mb pdf - need alot of time to run this script for 1 rdf
        }
        foreach ($this->xpathDom->query("/Digest/object[@name=\"" . $objectName . "\"]/head_spans/span") as $headSpan)
        {
            $result[] = $headSpan;
            $headStartEnd[intval($headSpan->getAttribute("start"))] = intval($headSpan->getAttribute("end"));
        }
        //check if for some reason any of the full_spans doesn't overlap with head_spans (might occur in anaphora)
        foreach ($this->xpathDom->query("/Digest/object[@name=\"" . $objectName . "\"]/full_spans/span") as $fullSpan)
        {
            $fullStart = intval($fullSpan->getAttribute("start"));
            $fullEnd = intval($fullSpan->getAttribute("end"));
            $found = false;
            foreach ($headStartEnd as $headStart => $headEnd)
            {
                if ($headEnd >= $fullStart && $headStart <= $fullEnd)
                {
                    $found = true;
                    break;
                }
            }
            if (! $found)
            {
                $result[] = $fullSpan;
                $headStartEnd[intval($fullSpan->getAttribute("start"))] = intval($fullSpan->getAttribute("end"));
            }
        }
        return $result;
    }

    
    /**
     * checkLangOverRide - checks if the lang override is exists
     * @param  $table - to check if the table of the element isn't language only
     * @return lang_mapper($row->value)
     */
    public function checkLangOverRide($table){
        global $isLangOverride;
                
        //Check if the flag is on, if the lang table already has the docid
        if($isLangOverride && $table == 'languages'){
            $this->mysqli;
            $res = sqlQuery($this->mysqli, "SELECT value FROM lang WHERE docId=$this->dcid");
            $row = $res->fetch_object();
            if(!empty($row->value)){
                // update the languages with the relevant lang
                return  lang_mapper($row->value);
            }
        }
        return null;
    }
    
    
    /**
     * update_date_field - setting the date from the meta data to be updated
     * @return $date_val - the date value
     */
    public function update_date_field(){
        //determine doc date by precedence: date_field, metaParameter date, file date attributes
        $override_time_values = ["file_creation_date", "file_last_modified_date", "file_last_accessed_date",'scan_time'];
      
        // 1 - only if there is no detected date or metadata date at date_field - use file override_time_values properties
        foreach ($override_time_values as $loop_element)
        {
            if (array_key_exists($loop_element, $this->fileFields) && !empty($this->fileFields[$loop_element])){
                $date_val = $this->fileFields[$loop_element];
                break;
            }
        }
        
        // 2 - if nothing is set - return the current time.
        if(empty($date_val)){
            $date_val = date('Y-m-d H:i:s');
        }
        
        // 3 - last - if date_val is updated - means we need to update date_field table
        $query = "INSERT INTO date_field (docId,value)
    			  	  VALUES ($this->dcid,'$date_val')";
        sqlQuery($this->mysqli,$query);
        
        
        
        // return $date_val - could be from steps 2/3
        return $date_val;
    }
    
    /**
     * update_date_field_extraData
     * @return $date_field - if not empty - return the date field in the right format
     */
    public function update_date_field_extraData(){
        // 1  - check if the file has a date setup at the date_field table - if so - dont do anything
        $res = sqlQuery($this->mysqli, "SELECT value FROM date_field WHERE docId = $this->dcid LIMIT 1");
        $row = $res? $res->fetch_object() : NULL;
        $date_field = $row? $row->value : NULL;
        //if date value is set at date_field - return it to be written in the files date
        if(!empty($date_field)){
            return prase_datefield_date($date_field);
        }
    }
    
    //
    /**
     * insertIntoTables - insert the values to the columns of the table
     * @param  $table - the table we gona insert the values to
     * @param  $columns - the column
     * @param  $values - the values
     */
    public function insertIntoTables($table, $columns, $values){
        $table = TablesInformation::getTableName($table);
        $sql = "INSERT INTO $table $columns VALUES ";
        $sql .= join(',',$values);
        sqlQuery($this->mysqli, $sql);
    }
    
    //get_parent_cols_vals($son_columns, $son_values) - get the parent table values - by comparing the relevant son_cols to the fixed cols
    // and get the relevant son_values out of it
    /**
     *  get_parent_cols_vals get the parent table values - by comparing the relevant son_cols to the fixed cols and get the relevant son_values out of it
     * @param  $son_columns - the son element columns
     * @param  $son_values - the son element values
     * @param  $parent_table - the parent table that we will fetch from it
     * @param  $parent_cols - current perant cols
     * @return $ parent cols and vals
     */
    public function get_parent_cols_vals($son_columns, $son_values, $parent_table, $parent_cols, $clean_name){
        $vals_return = array();
        //remove for each values array the indexes that wont be relvant
        foreach($son_values as $val){
            $values_trim = substr($val, 1, -1);
            $values_trim = str_replace('"', "'", $values_trim);
            $val_arr = str_getcsv($values_trim, ",", "'");
            $parent_table = TablesInformation::getTableName($parent_table);
            $val_arr[0] = $this->getConceptId($parent_table); //change the concept_type index to the parent      
            $val_arr[1] = $this->getConceptId($parent_table); //change the concept_type_source index to the parent   
            $val = "('".join("','",$val_arr)."')";
            $vals_return[] = $val;
            
        }
        return ['columns'=>"$son_columns", 'values'=>$vals_return];
        
        
    }
//     public function get_parent_cols_vals($son_columns, $son_values, $parent_table, $parent_cols, $clean_name){
//         global $entitiesArray;
//         $parent_cols = explode(',', str_replace(' ', '', $parent_cols)); //takes the parent cols and make it an array
//         $i = 0;
//         $exclude_index_arr = []; //we will insert here the indexes of non-relvant columns
//         $new_cols = []; // final return value 1
//         $new_vals = []; // final return value 2
//         $parent_table_name = TablesInformation::getTableName($parent_table);
//         $son_columns = substr($son_columns, 1, -1); //removes ( and )
//         $son_col_arr = explode(',',$son_columns);
//         $isEntity = false;
//         //if it's entity mark the name/orgtext with _version
//         if(array_key_exists(strtolower(str_replace("-","_",$parent_table)), $entitiesArray) !==false || array_key_exists($parent_table, $entitiesArray)!==false){
//             $isEntity = true;
//         }
        
//         //find columns that does not appear in the fixed cols - and insert their indexes to exclude_index_arr
//         foreach($son_col_arr as $col){
//             $col_inner_arr = explode('.',$col);
//             if(count($col_inner_arr)>1){
//                 $col_name = trim($col_inner_arr[1]);
//             }
//             else{
//                 $col_name = $col_inner_arr[0];
//             }
//             if($isEntity && ($col_name == 'name' || $col_name == 'orgText')){
//                 $col_name .= '_version';
//             }
//             if(!in_array($col_name,$parent_cols)){
//                 $exclude_index_arr[] = $i;
//             }
//             else{
//                 $new_cols[] = $parent_table_name.'.'.$col_name;
//             }
//             $i++;
//         }
        
//         //remove for each values array the indexes that wont be relvant
//         foreach($son_values as $val){
//             $values_trim = substr($val, 1, -1);
//             $values_trim = str_replace('"', "'", $values_trim);
//             $val_arr = str_getcsv($values_trim, ",", "'");
//             foreach($exclude_index_arr as $null_index){
//                 unset($val_arr[$null_index]);
//             }
//             $val_arr = array_values($val_arr);
//             for($i=0; $i < count($val_arr); $i++){
//                 if(is_numeric ($val_arr[$i])){
//                     continue;
//                 }
//                 $val_arr[$i] = "'" . trim($val_arr[$i],"\'") . "'";
//             }
//             if($parent_table == 'Organizational-identity'){
//                 $new_cols[] = "$parent_table_name.clean_name";
//                 $val_arr[] = "'$clean_name'";
//             }
            
//             $val = '('.join(',',$val_arr).')';
//             $new_vals[] = $val;
//         }
        
//         return ['columns'=>'('.join(',',$new_cols).')', 'values'=>$new_vals];
//     }
    
    /**
     * setMetaDataFolder
     */
    public function setMetaDataFolder(){
        $mkDirRetryCount = 0;
        $directory = getDocMetadataFolder($this->dcid);
        
        is_dir($directory) ||  mkdir_full($directory, 0777, true);
        
        //retry making directory (in case of network problems)
        while (!is_dir($directory))
        {
            if (++$mkDirRetryCount > 10){
                return;
            }
            
            write_to_log("ERROR", "Failed creating directory. Sleeping and retrying. count: " . $mkDirRetryCount);
            sleep(2);
            mkdir_full($directory, 0777, true);
        }
        return $directory;
    }
    
    
    //idea taken from how-to-iterate-utf-8-string-in-php
    public function nextchar($string, &$pointer){
        if(!isset($string[$pointer]))
        {
            return false;
        }
        $char = ord($string[$pointer]);
        if($char < 128){
            return $string[$pointer++];
        }else{
            if($char < 224){
                $bytes = 2;
            }elseif($char < 240){
                $bytes = 3;
            }elseif($char < 248){
                $bytes = 4;
            }elseif($char == 252){
                $bytes = 5;
            }else{
                $bytes = 6;
            }
            $str =  substr($string, $pointer, $bytes);
            $pointer += $bytes;
            return $str;
        }
    }
    
    
    
    
    public function compareTitle($a, $b)
    {
        if ($a['title'] == $b['title']) {
            return 0;
        }
        return ($a['title'] < $b['title']) ? -1 : 1;
    }
    
    public function getInstanceOrgText(&$digestInstance, &$name)
    {
     
        $longestText = "";
        if (key_exists("children", $digestInstance)){
            for ($k = 0; $k < sizeof($digestInstance["children"]); $k++){
                if(key_exists("children", $digestInstance["children"][$k])){
                    for ($t = 0; $t < sizeof($digestInstance["children"][$k]["children"]); $t++){
                        $curr_text = $this->arr_word[$digestInstance["children"][$k]["children"][$t]["span_key"]];
                        $name = $digestInstance["children"][$k]["children"][$t]["title"];
                        
                        if ((!empty($curr_text) && empty($longestText)) ||
                            ((!empty($curr_text) && !empty($longestText)) && (strlen($curr_text) > strlen($longestText)))){
                                $longestText = $curr_text;
                        }
                    }
                }else{
                    $curr_text = $this->arr_word[$digestInstance["children"][$k]["span_key"] ?? $digestInstance["children"][$k]["key"]]?? null;
                    $name = $digestInstance["children"][$k]["title"];
                    
                    if ((!empty($curr_text) && empty($longestText)) ||
                        ((!empty($curr_text) && !empty($longestText)) && (strlen($curr_text) > strlen($longestText)))){
                        $longestText = $curr_text;
                    }
                }
                
            }
        }
        else{
            $curr_text = $this->arr_word[$digestInstance["span_key"] ?? $digestInstance["key"] ] ?? null;
            $name = $digestInstance["title"];
            if ((!empty($curr_text) && empty($longestText)) || 
                ((!empty($curr_text) && !empty($longestText)) && (strlen($curr_text) > strlen($longestText)))){
                    $longestText = $curr_text;
            }
        }
        return $longestText;
    }
    
    public function addOccurrences(){
        write_to_log("TRACE", $this->dcid . " Background: add occurrences");
        
        //add occurrences
        for ($i = 0; $i < sizeof($this->digestJson); $i++){
            for ($j = 0; $j < sizeof($this->digestJson[$i]['children']); $j++){
                if(is_array($this->digestJson[$i]['children'][$j]['title'])){
                    $this->digestJson[$i]['children'][$j]['title'] = reset($this->digestJson[$i]['children'][$j]['title']);
                }
                
                if (array_key_exists('children',$this->digestJson[$i]['children'][$j]) && sizeof($this->digestJson[$i]['children'][$j]['children']) > 0){
                    $curr_title = $this->digestJson[$i]['children'][$j]['title'];
                    $curr_key = $this->digestJson[$i]['children'][$j]['key'];
                    
                    $this->digestJson[$i]['children'][$j]['title'] .= " (" . sizeof($this->digestJson[$i]['children'][$j]['children']) . ")";
                    
                   
                    //if Key of value exists in orgTextKeyArr (and > "")
                    if (array_key_exists($curr_key, $this->orgTextKeyArr) && strlen($this->orgTextKeyArr[$curr_key]) > 1){
                        $this->digestJson[$i]['children'][$j]['orgText'] = $this->orgTextKeyArr[$this->digestJson[$i]['children'][$j]['key']] . " (" . sizeof($this->digestJson[$i]['children'][$j]['children']) . ")";    
                    }
                    
                    //else - if title exists in orgTextValueArr
                    elseif (array_key_exists($curr_title, $this->orgTextValueArr)){
                        $this->digestJson[$i]['children'][$j]['orgText'] = $this->orgTextValueArr[$curr_title] . " (" . sizeof($this->digestJson[$i]['children'][$j]['children']) . ")";
                    }
                    else{                        
                        $this->digestJson[$i]['children'][$j]['orgText'] = $this->digestJson[$i]['children'][$j]['title'] . " (" . sizeof($this->digestJson[$i]['children'][$j]['children']) . ")";
                    }
                }else{
                    if(empty($this->digestJson[$i]['children'][$j]['key'])){
                        write_to_log("ERROR", "no key in addOccurrences for ".json_encode($this->digestJson[$i]['children'][$j],true));
                        continue;
                    }
                    if (array_key_exists($this->digestJson[$i]['children'][$j]['key'], $this->orgTextKeyArr) && strlen($this->orgTextKeyArr[$this->digestJson[$i]['children'][$j]['key']]) > 1){
                        $this->digestJson[$i]['children'][$j]['orgText'] = $this->orgTextKeyArr[$this->digestJson[$i]['children'][$j]['key']];
                    }
                        elseif	(array_key_exists($this->digestJson[$i]['children'][$j]['title'], $this->orgTextValueArr)){
                            $this->digestJson[$i]['children'][$j]['orgText'] = $this->orgTextValueArr[$this->digestJson[$i]['children'][$j]['title']];
                        }
                        else{
                            $this->digestJson[$i]['children'][$j]['orgText'] = $this->digestJson[$i]['children'][$j]['title'];
                        }
                }
                $k = 0;
                $arr = array();
                if (array_key_exists("children", $this->digestJson[$i]['children'][$j])){
                    foreach ($this->digestJson[$i]['children'][$j]['children'] as $key => $val){
                        if(is_array($this->digestJson[$i]['children'][$j]['children'][$key]['title'])){
                            $this->digestJson[$i]['children'][$j]['children'][$key]['title'] = reset($this->digestJson[$i]['children'][$j]['children'][$key]['title']);
                        }
                        if (strlen($this->digestJson[$i]['children'][$j]['children'][$key]['title'] ?? "")>1){
                            if(is_array( $val['title'])){
                                $val['title'] =  reset($val['title']);
                            }
                            if (array_key_exists($val['key'], $this->orgTextKeyArr) && strlen($this->orgTextKeyArr[$val['key']]) > 1){
                                $val['orgText'] = $this->orgTextKeyArr[$val['key']];
                            }elseif (array_key_exists($val['title'], $this->orgTextValueArr)){
                                $val['orgText'] = $this->orgTextValueArr[$val['title']];
                            }
                            else{
                                $val['orgText'] = $val['title'];
                            }
                            $arr[$k++] = $val;
                        }
                        unset($this->digestJson[$i]['children'][$j]['children'][$key]);
                    }
                }
                for ($t = 0; $t < $k; $t++){
                    $this->digestJson[$i]['children'][$j]['children'][$t] = $arr[$t];
                }
            }
            if ($j > 0){
                if (array_key_exists($this->digestJson[$i]['key'], $this->orgTextKeyArr) && strlen($this->orgTextKeyArr[$this->digestJson[$i]['key']]) > 1){
                    $this->digestJson[$i]['orgText'] = $this->orgTextKeyArr[$this->digestJson[$i]['key']] . " (" . $j . ")";
                }elseif (array_key_exists($this->digestJson[$i]['title'], $this->orgTextValueArr)){
                    $this->digestJson[$i]['orgText'] = $this->orgTextValueArr[$this->digestJson[$i]['title']] . " (" . $j . ")";
                }
                $this->digestJson[$i]['title'] .= " (" . $j . ")";
            }
            
        }
        write_to_log("TRACE", $this->dcid . " Background: add occurrences - finished");
        
    }
    
    public function get_relationship_type($value) : string {
        global $relationTypeArr;
        $rel_type = unqualifyName(toDbString($this->mysqli, $value));
        $caption = $this->qname2node["dig"][$value]['iv:caption']['en'] ?? null;
        return $caption ?? (array_key_exists($rel_type, $relationTypeArr)? $relationTypeArr[$rel_type] : $rel_type);
    }
    
    
    //if the type is a reference to a kb instance return the original kb type
    public function follow_if_reference_type($type, $digest): string {
        $follow_rel_type = null;
        if (isset($digest) && array_key_exists($type, $digest) && isset($digest[$type]["iv:instance"])) {
            $follow_rel_type = $digest[$type]["iv:instance"];
        }
        return $follow_rel_type??$type;
    }
    
    //sentiment and parameters
    public function getSentInstType($sentInstQName)
    {
        global $annotElementsSubclasses;
        $sentInstType = $this->qname2node["dig"][$sentInstQName]["rdf:type"];
        if (!is_array($sentInstType)){
            $sentInstType = [$sentInstType];
        }
            
            foreach ($sentInstType as $v){
                $s = unqualifyName($v);
                foreach (["Sentiment-indicator-type", "Anti", "Pro"] as $sent){
                    if(array_key_exists($sent, $annotElementsSubclasses) && is_array($annotElementsSubclasses[$sent]) && in_array($s,$annotElementsSubclasses[$sent])){
                        return $s;
                    }
                }
            }
            write_to_log("TRACE", "None of SentInstType is not found in sentiment subclasses. Taking first:" . print_r($sentInstType, true));
            if (count($sentInstType)){
                return unqualifyName($sentInstType[0]);
            }
                
                return NULL;
    }
    
    
    
    public function parentClassHandler($conceptName, $concept, $name, $clean_name = null, $alerts_concepts_arr = []){
        global $entitiesArray;
        if(!isset($this->table2values[$conceptName])){
            return ;
        }
        $curr_son = $this->table2values[$conceptName];
        
        $curr_son_cols = key($curr_son); // getting the columns of the son element
        
        $curr_son = array_shift($curr_son); //getting the keys and values of the son
        
        $curr_son_values = $curr_son["values"]; //getting the values for the son
        
        if(isset($curr_son) && isset($this->subClass2classes[$concept['key']])){
            foreach($this->subClass2classes[$concept['key']] as $parent_table){
        
                
                $parent_table = strtolower($parent_table);
                if(!array_key_exists(($parent_table),$this->merged_data)){
                    continue;
                }
                
            
                //getting the total columns of the parent from the merged data (for filtering the values&cols of the son table)
                $parent_sql_cols = TablesInformation::getElementsColumns($this->mysqli, $parent_table);
                
                //getting the parent columns and values for inserting to the db
                $parent_cols_vals = $this->get_parent_cols_vals($curr_son_cols, $curr_son_values, $parent_table,$parent_sql_cols, $clean_name);
                $parent_cols = explode(',', trim($parent_cols_vals['columns'], '()'));
                $parent_vals = $parent_cols_vals['values'];
                foreach($parent_vals as &$val_element){
                    $val_element = explode(',', trim($val_element, '()'));  
                }

                //check for manage entities if parent table has it
                $ceaggTable = str_replace('-', '_', $parent_table);
                
                if(in_array($ceaggTable, $alerts_concepts_arr)){
                    if(!array_key_exists($parent_table, $alerts_concepts_arr['concepts'])){
                        $this->alertsArray['concepts'][$parent_table] = [];
                    }
                    if(!in_array($name,  $this->alertsArray['concepts'][$parent_table])){
                        $this->alertsArray['concepts'][$parent_table][] = $name;
                    }
                }
                
//                 if(array_key_exists($ceaggTable, $entitiesArray)){
//                     $ceagg_arr = $this->checkInAggOrCross($ceaggTable, $name);
                  
                    
//                     if(!empty($ceagg_arr['name_agg'])){
//                         $parent_cols[] = "names_agg";
//                         foreach($parent_vals as &$val_element){
//                             $val_element[] = "\"$ceagg_arr[name_agg]\"";
//                         }
//                     }
//                     if(!empty($ceagg_arr['destConcept'])){
//                         $concept_index = array_search('concept_type', $parent_cols);
//                         $concept_id = $this->getConceptId($ceagg_arr['destConcept']);
                            
//                             $val_element[$concept_index] = $this->getConceptId($ceagg_arr['destConcept']);
                        
//                     }
//                 }
                
                $parent_cols = "(".implode(',', $parent_cols).")";
                foreach($parent_vals as &$val_element){
                    $val_element = "(".implode(',', $val_element).")";
                }
                
                
                //1 - if the parent table does not exists in the table2value yet - create new element and add it
                if(!array_key_exists(TablesInformation::getTableName($parent_table), $this->table2values)){
                    
                    $parent_element =  [$parent_cols=>
                        ["keys"=>$curr_son['keys'],
                            "values" => $parent_vals]
                    ];
                    $this->table2values[TablesInformation::getTableName($parent_table)] = $parent_element;
                }
                //2 - if the parent table does exists in the table2value - add the values and keys to it
                else{
                    //go over the curr_elements keys and the parent_col_vals values and place them
                    // loop over the curr_son keys and parent_cols_vals values - since they are match to eachother
                    $parent_columns_exists = key($this->table2values[TablesInformation::getTableName($parent_table)]);
                    $parent_exists_keys = $this->table2values[TablesInformation::getTableName($parent_table)][$parent_columns_exists]['keys'];
                    
                    for($i = 0; $i < count($curr_son['keys']); $i++){
                        $key_input = $curr_son['keys'][$i];
                        $value_input = $parent_cols_vals['values'][$i];
                        if(!in_array($key_input,$parent_exists_keys)){
                            $this->table2values[TablesInformation::getTableName($parent_table)][$parent_cols]['keys'][] = $key_input;
                            $this->table2values[TablesInformation::getTableName($parent_table)][$parent_cols]['values'][] = $value_input;
                            
                            //keep track of non entities concepts for emotion calculation
                            if(strtolower($parent_table) == 'concept' && !array_key_exists($conceptName, $entitiesArray)){
                                $this->total_concepts++;
                            }
                        }
                    }
                    
                }
            }
        }

    }
    
    public function getTableFromUrl($url){
        if(is_array($url)){
            $url = $url[0];
        }
        $arr = explode('#', $url);
      
        $res = strtolower(str_replace('-', '_', $arr[1]));
        
        return $res;
    }
    
    public function getRelationsInstanceCaption($relation){
        $type_item = $relation['iv:relationship-type'];
        $rel = $this->qname2node["dig"][$type_item] ?? null;
        $res = !empty($rel) && array_key_exists("iv:caption", $rel) &&  array_key_exists("iv:name", $rel) ?
                                ["caption" => $rel['iv:caption']['en'], "inst" => $rel["iv:name"]] : null;
        return $res;
    }
    
    public function jsonRelationCreate(){
        $json = array();
        $obj = null;
        $interEntRel = qualifyName("Inter-entity-relationship", "ont");
        if (array_key_exists($interEntRel, $this->pred2inst["dig"]["rdf:type"])){
            foreach ($this->pred2inst["dig"]["rdf:type"][$interEntRel] as $relation)
            {
      
                $relId = $relation["rdf:about"] ?? null;
                $se = $relation["iv:subject-entity"] ?? null;
                $oe = $relation["iv:object-entity"] ?? null;
                $subj = $this->qname2node["dig"][$se]["iv:caption"] ?? null;
                if (is_array($subj)){
                    $subj = $subj["en"] ?? "";
                }
                if (is_array($oe))
                {
                    write_to_log("TRACE", "relation object is an array, taking first item: " . print_r($oe, true));
                    $oe = $oe[0];
                }
                if($oe){
                    $obj = $this->qname2node["dig"][$oe]["iv:caption"] ?? null;
                    if (is_array($obj)){
                        $obj = $obj["en"];
                    }
                }
             
                
                //if relationship-type is an array create multiple jsons
                if(is_array($relation["iv:relationship-type"])){

                    foreach($relation["iv:relationship-type"] as $curr_rel_type){
                        $temp_rel = $relation;
                        $temp_rel['iv:relationship-type'] = $curr_rel_type;
                        $rel_type_value = $this->getRelationsInstanceCaption($temp_rel) ?? $this->follow_if_reference_type($curr_rel_type, $this->qname2node["dig"]);
                        $rel_type_inst = $rel_type_value["inst"] ?? null;
                        $rel_type = $rel_type_value["caption"] ?? null;
                        $json['results']['bindings'][] = array(
                            'relId' => array('type'=>"uri", 'value' => $relId),
                            'subj' => array('type'=>"literal", "xml:lang" => "en", 'value' => $subj),
                            'obj'  => array('type'=>"literal", "xml:lang" => "en", 'value' => $obj),
                            'se' => array('type'=>"uri", 'value' => $se),
                            'rel_type_inst' => $rel_type_inst,
                            'rel_type' => $rel_type,
                            'oe' => array('type'=>"uri", 'value' =>$oe)
                        );
                    }
                }
                //if relationship-type is an single    
                else{     
                    $rel_type_value = $this->getRelationsInstanceCaption($relation) ?? $this->follow_if_reference_type($relation["iv:relationship-type"], $this->qname2node["dig"]);                
                    $rel_type_inst = $rel_type_value["inst"] ?? null;
                    $rel_type = $rel_type_value["caption"] ?? null;
                    
                    $json['results']['bindings'][] = array(
                        'relId' => array('type'=>"uri", 'value' => $relId),
                        'subj' => array('type'=>"literal", "xml:lang" => "en", 'value' => $subj),
                        'obj'  => array('type'=>"literal", "xml:lang" => "en", 'value' => $obj),
                        'se' => array('type'=>"uri", 'value' => $se),
                        'rel_type_inst' => $rel_type_inst,
                        'rel_type' => $rel_type,
                        'oe' => array('type'=>"uri", 'value' =>$oe)
                    );
                }
            }
        }
        dumpDocsDbg("relationsJson", $this->dcid,  $json);
        return $json;
        
    }
    
    public function setTable2valuesPhrases($ngrams) {
        $this->table2values["phrases"] = $ngrams;
    }
    
    public function getTagsArr() {
        $sql = "SELECT results FROM files WHERE id='$this->dcid'";
        $res = sqlQuery($this->mysqli, $sql);
        $row = $res->fetch_object();
        $results = $row->results ?? null;
        return $results;   
    }
    
    /*
     * get pie/trend here for document
     * than add it to "inc_dashboard"
     * 
     */
    public function inc_calc(){        
        $inc_bool = getSystemSettingsProp($this->mysqli, "inc_calc");
        require_once 'demosettings.php';
        require_once 'v2'.DIRECTORY_SEPARATOR.'mainEng_funcs_v2.php';
        if(!$inc_bool){
            write_to_log("TRACE", "manual update for panels - please go to dashboard and click on refresh when needed");     
            return null;
        }
        $panels = getPanelsConfiguration($this->mysqli, $this->username, "dashboard", "1");
        foreach ($panels['panelsConfiguration'] as $dashboard_element){
            
            $document_res = [];
            
            $table = $dashboard_element["name"];
            write_to_log("TRACE", "Incremntal process on $table");
            
            $pieData = getTableElementData($this->mysqli, 'dashboard', $table, 'username', $this->username);          
            $pieDetailsArray = getPieDetails($pieData);
    
            $_REQUEST["username"] = $this->username;
            $_REQUEST["isAdmin"] =   getUserPermission(CAN_SHOW_ALL_USER, $this->username);
            include_once 'StatisticPageEng.php';
     
            $document_res = getPie($this->mysqli, true,  $pieDetailsArray,  null, $this->dcid);           
            write_to_log("TRACE", "Fetched document $this->dcid getPie of $table");
            
            $graphType =  $pieData->type ?? "";
              
            $cache_key =  CacheEng::generate_cache_key($this->mysqli, $pieData->type ?? null, $table, $this->username);
          
            $this->update_panel($cache_key, $document_res);
            
            /**
             * if its a private user - needs to update the ADMIN rows in the cache as well
             */
            if(!$_REQUEST["isAdmin"]){
                $_REQUEST["isAdmin_override"] = true;
                $cache_key_admin =  CacheEng::generate_cache_key($this->mysqli, $pieData->type ?? null, $table, $this->username);
                $document_res_admin = getPie($this->mysqli, true,  $pieDetailsArray,  null, $this->dcid);
                
                $this->update_panel($cache_key_admin, $document_res_admin);
                $_REQUEST["isAdmin_override"] = false;
                
            }

        }
        //update the overview
        $cache_key =  CacheEng::generate_cache_key($this->mysqli, 'statistics', 'key_stats',  $this->username);
        $sql = "SELECT id from cacheeng WHERE cache_key = \"$cache_key\"";
        $res = sqlQuery($this->mysqli, $sql);
        $row = $res->fetch_object();
        
        $overview_stats = caseStatistics($this->mysqli, true, $this->username, $_REQUEST["isAdmin"]);
        
        $overview_stats = $this->mysqli->real_escape_string(json_encode($overview_stats, true));
        
        //if empty - just insert to table
        if(empty($row->id)){
            $sql = "INSERT INTO cacheeng (cache_key, results) VALUES(\"$cache_key\", \"$overview_stats\")";
        }
        else{
            $sql = "   UPDATE cacheeng SET  results = \"$overview_stats\"  WHERE cache_key = \"$cache_key\"";
        }
 
        $res = sqlQuery($this->mysqli, $sql);
        
        /**
         * if its a private user - needs to update the ADMIN rows in the cache as well
         */
        if(!$_REQUEST["isAdmin"]){
            $_REQUEST['isAdmin_override'] = true;
            $cache_key =  CacheEng::generate_cache_key($this->mysqli, 'statistics', 'key_stats',  $this->username);
            $overview_stats = caseStatistics($this->mysqli, true, $this->username, true);
            $overview_stats = $this->mysqli->real_escape_string(json_encode($overview_stats, true));
            
            //if empty - just insert to table
            $sql = "SELECT id from cacheeng WHERE cache_key = \"$cache_key\"";
            $res = sqlQuery($this->mysqli, $sql);
            $row = $res->fetch_object();
            if(empty($row->id)){
                $sql = "INSERT INTO cacheeng (cache_key, results) VALUES(\"$cache_key\", \"$overview_stats\")";
            }
            else{
                $sql = "   UPDATE cacheeng SET  results = \"$overview_stats\"  WHERE cache_key = \"$cache_key\"";
            }
            
            $res = sqlQuery($this->mysqli, $sql);
            $_REQUEST["isAdmin_override"] = false;
            
        }
        
    }
    
    /**
     * update panel action
     * @param unknown $cache_key
     * @param unknown $document_res
     */
    public function update_panel($cache_key, $document_res){
        $sql = "SELECT id from cacheeng WHERE cache_key = \"$cache_key\"";
        $res = sqlQuery($this->mysqli, $sql);
        $row = $res->fetch_object();
        
        //if empty - just insert to table
        if(empty($row->id)){
            $json_doc = $this->mysqli->real_escape_string(json_encode($document_res, true));
            $sql = "INSERT INTO cacheeng (cache_key, results) VALUES(\"$cache_key\", \"$json_doc\")";
            $res = sqlQuery($this->mysqli, $sql);
        }
        
        //update the new values to the cacheng table
        elseif(!empty($document_res)){
            foreach($document_res as $section=>$values){
                if(in_array($section, array('title','total','savedSearchName','FilesWithoutObjects_num'))){
                    continue;
                }
                foreach($values as $ont_key=>$item){
                    $curr_scale = $item["scale"];
                    $curr_ont = $this->mysqli->real_escape_string($item["ont"] ?? $item["label"]);
                    $curr_label = $this->mysqli->real_escape_string($item["label"]);
                    $curr_org = array_key_exists("orgText", $item) ? $this->mysqli->real_escape_string($item["orgText"]) : null;
                    $orgText_str = empty($curr_org) ?  "" : "'orgText', \"$curr_org\",";
                    //in case of key not in json - it'll add empty one with the same ont/name
                    $sql ="UPDATE cacheeng SET results = JSON_SET(results, '$.".$section.".\"".$ont_key."\"', COALESCE(JSON_EXTRACT(results, '$.".$section.".\"".$ont_key."\"'),
                    JSON_OBJECT('ont',\"$curr_ont\", 'label', \"$curr_label\", $orgText_str 'scale', 0)))
                    WHERE cache_key = \"$cache_key\"";
                    
                    $res = sqlQuery($this->mysqli, $sql);
                    
                    $sql ="UPDATE cacheeng SET results = JSON_SET(results, '$.".$section.".\"".$ont_key."\".scale', JSON_EXTRACT(results, '$.".$section.".\"".$ont_key."\".scale') + $curr_scale)
                    WHERE cache_key = \"$cache_key\"";
                    
                    $res = sqlQuery($this->mysqli, $sql);
                    
                }
                
                //updates the total
                $sql = "   UPDATE cacheeng SET  results = JSON_SET(results, '$.total',
                    (SELECT * FROM  (SELECT JSON_LENGTH(results->\"$.amount\") FROM cacheeng
                        WHERE cache_key = \"$cache_key\") AS t))
                        WHERE cache_key = \"$cache_key\"";
                    $res = sqlQuery($this->mysqli, $sql);
                    
            }
        }
    }
    
    
    function getManageEntities(){
        $response = ["persons"=>[], "organisations"=>[], "events"=>[], "places"=>[]];
        $sql_user = " TRUE ";//$this->isAdmin  ? " TRUE " : "  username = '$this->username'";
        $sql = "SELECT * FROM entitiesmanage WHERE entity<>\"smart\" AND $sql_user";
        $res = sqlQuery($this->mysqli, $sql);
        if (!empty($res)){
            while ($row = $res->fetch_object()) {
                $e = array_filter(explode(";",$row->aggregatedName));
                $response[$row->entity][] = array("Name" => $row->name, "Aggregated" => array_values($e));
            }
        }
        return $response;
    }
    
    function getHiddenEntities(){
        $response = [];
        $sql = "SELECT entity,names FROM hide_entities";
        $res = sqlQuery($this->mysqli, $sql);
        if (!empty($res)){
            while($obj = $res->fetch_object()){
                $response[$obj->entity] = explode('|', $obj->names);
            }
        }
        return $response;
    }
    
    /**
     * checkHiddrenEntities - checks if the element supposed to be hidden from the DB (means - do not insert it)
     * @param unknown $type - the concept 
     * @param unknown $name - the name
     * @return boolean
     */
    function checkHiddrenEntities($type, $name){
        $res = false;
        if(array_key_exists($type, $this->hiddenEntitiesArr)){
            if(in_array($name, $this->hiddenEntitiesArr[$type])){
                $res = true;
            }
        }
        return $res;
    }
    
    function checkInAggOrCross($type, $name){
        global $manageEntitiesArray;
        $destConcept = null;
        $name_agg = null;
        
        //A - get crossentities
        if(is_array($name)){
            $name = $name[0];
        }
        $name = $this->mysqli->escape_string($name);
        $ce_sql = "SELECT destConcept FROM crossentities WHERE entity=\"$name\" AND orgConcept=\"$type\"";// AND username=\"$this->username\"";
        $res = sqlQuery($this->mysqli, $ce_sql);
        if($res && $row = $res->fetch_object()){
            $destConcept = $row->destConcept;
        }
        
        //B - get manageEntitiesArr
        $tmp_type = $destConcept ?? $type; //assign the original type - or the cross entities type
        $type_em =  array_key_exists($tmp_type, $manageEntitiesArray) ? $manageEntitiesArray[$tmp_type] : $tmp_type;
        $tmp = $this->manageEntitiesArr;
        if(!empty($this->manageEntitiesArr[$type_em])){
            foreach($this->manageEntitiesArr[$type_em] as $element){
                if(in_array($name, $element['Aggregated'])){
                    $name_agg = $element["Name"];
                }
            }
        }
        
       

        return ["name_agg" => $name_agg, "destConcept" => $destConcept];
    }

} 



Class InsertArray {
    public $insert_array = NULL;
    public $mysqli = null;
    
    public function __construct($mysqli){
        $this->insert_array = array();
        $this->mysqli = $mysqli;
    }
    
    public function add_element($concept, $keys, $values){
        write_to_log("TRACE", "Adding to insertArray: Concept - $concept, keys - $keys, values - $values");
        
        $keys = str_replace(' ', '', $keys);
        $keys = trim($keys, '()');
        $values = trim($values, '()');
        
        $this->insert_array[$concept][$keys][] = $values;
        
    }
    
    public function insert_to_DB($dcid){
        dumpDocsDbg("InsertArray", $dcid, $this->insert_array);
        foreach($this->insert_array as $concept=>$keys_values){
          
            foreach($keys_values as $keys=>$values_arr){
                foreach ($values_arr as &$value) {
                    $value = '('.$value.')';
                }
                $values =  implode(', ',$values_arr);
                
                $final_sql = "INSERT INTO $concept ($keys) VALUES $values";
                sqlQuery($this->mysqli, $final_sql);
                

            }
        }
    }
    

}

