<?php

//============================================================================
//==========================DATA UTILITIES====================================
//============================================================================
require_once 'validation.php';
/**
 *
 * @param unknown $wl_mysql
 * @param unknown $searchedPerson
 * @param unknown $person_data
 * @param unknown $wlSelected
 * @param unknown $rule_arr
 * @param number $id
 * @return number[]|unknown[]|array[][]|unknown[][]
 */
function searchPerson($wl_mysql, $searchedPerson, $searched_person_data, $wlSelected, $rule_arr, $columns_metadata, $id = 0){
    $matches = findPossibleMatches($wl_mysql, $searchedPerson, $searched_person_data, $wlSelected, $rule_arr, $columns_metadata, $id);
    return $matches;
}

/**
 * 
 * @param unknown $rule_data
 * @param unknown $field
 * @param unknown $na
 * @return string|unknown
 */
function getCurrData($rule_data, $field, $na){  
    $curr_data = "";
    if(array_key_exists($field, $na)){
        if(is_array($na[$field])){
            if(in_array($field, $rule_data['initials_tables'])){
                $curr_data = $na[$field]["caption"];
            }
            else{
                $curr_data =  $na[$field]["instance"];           
          }
        }
        else{
            $curr_data = $na[$field];
        }
      
    }
    return $curr_data;
}


//for pssible same person gn OR fn - check if its single NC name alternative - return true - otherwise - false
function isSingleNCinNA($na){
    if(array_key_exists("given_name", $na) xor array_key_exists("family_name", $na)){
        if(array_key_exists("given_name", $na) && !array_key_exists("family_name", $na)){
            unset($na["given_name"]);
            foreach($na as $f=>$v){
                if(strpos($f, "_name") !== false){
                    return false;
                }
            }
        }
        if(array_key_exists("family_name", $na) && !array_key_exists("given_name", $na)){
            unset($na["family_name"]);
            foreach($na as $f=>$v){
                if(strpos($f, "_name") !== false){
                    return false;
                }
            }
        }
        return true;
    }
    return false;
}

/**
 * extraValidationRules - check for specific rules some cases
 * @param unknown $matches_arr
 * @param unknown $rule
 */
function extraValidationRules($na, $matches_arr, $rule, $person_ref, $input){
    //check if its possible_same_person gn or fn - counts only if we have 1 NC
    if(in_array($rule,["possible_same_person_gn","possible_same_person_fn"])){
        if(!isSingleNCinNA($na)){
            return false;
        }
    }
    //if its possible same family - check if person ref already defined under possible same person
    if($rule=="possible_same_family"){
        if(array_key_exists("possible_same_person", $matches_arr)){
            if(array_key_exists($input, $matches_arr["possible_same_person"])){
                return false;           
            }
        }         
    }
    return true;  
}

/**
 * findPossibleMatches - find all possible matches
 * @param unknown $wl_mysql
 * @param unknown $searchedPerson
 * @param unknown $person_data
 * @param unknown $wlSelected
 * @param unknown $rule_arr
 * @param number $id
 * @return number[]|unknown[]|array[][]
 */
function findPossibleMatches($wl_mysql, $searchedPerson, $person_data, $wlSelected, $rule_arr, $columns_metadata, $id = 1){
    $matches_arr = [];
    /**
     * Go over all name alternative of the searchedPerson name alternative
     */
    foreach ($person_data["Name Alternatives"] as $na){
        //A - if NA is no analysis element - search exact match in full analysis table only and return
        if(isset($na["warning"]) && $na["warning"] == "No Analysis"){
            $matches_arr = getExactMatchData($wl_mysql, $na);
            $matches = [];
            if(!array_key_exists("possible_same_person", $matches_arr)){
                return;
            }
            $input = key($matches_arr["possible_same_person"]);
            $ids_arr = $matches_arr["possible_same_person"][$input];
            $matches["possible_same_person"][] =  ["ID" => implode(',', $ids_arr), "Name" => $input, "score" => 100];
                
            
            $results[] = ["ID" => $id, "Name" => $input, "Rules" => $matches];
            return $results;
            
        }
        
        //B - For regular NA - iterate over the rule array and find matching for that NA
        foreach($rule_arr as $rule=>$rule_data){
            $keys_arr = $rule_data["keys"];
            $mandatory_values_cond = true;
            
            /**
             * 1 - create matching keys for the rule & NA
             */
            $matching_keys_res = getMatchingKeys($wl_mysql, $na, $rule_data, $keys_arr);
            $matching_keys_arr = $matching_keys_res["matching_keys_arr"];
            $matching_keys_initial_arr = $matching_keys_res["matching_keys_initial_arr"];
            $mandatory_values_cond = $matching_keys_res["mandatory_values_cond"];
            
            //if NA has no mandatory values for the rule OR no matching keys - skip
            if(!$mandatory_values_cond){
                continue;
            }
            
            /**
             * 2 - query the sql table of that rule to look for match
             */    
            //keep matching_keys strings for the sql queries further 
            $matching_str = implode("','", $matching_keys_arr);
            $matching_str_initial = implode("','", $matching_keys_initial_arr);
            
            //keep boolean - if we found_match in a rule - no need to continue to look for initial rule match
            $found_match = false;
            
            //search for that key in the relevant rule table
            if(!empty($matching_str)){         
                $sql = "SELECT DISTINCT input, GROUP_CONCAT(DISTINCT person_ref) AS person_ref,GROUP_CONCAT(DISTINCT na_index) AS na_indexes
                         FROM full_analysis WHERE person_ref IN(SELECT DISTINCT person_ref FROM $rule 
                        WHERE rule_key IN (\"".$matching_str."\"))
                        GROUP BY input";
      
                $res = sqlQuery($wl_mysql, $sql);
                while($row = $res->fetch_object()){
                    if(!empty($row->input)){     
                        $person_ref = $row->person_ref;
                        $na_indexes = $row->na_indexes;
                        $input = $row->input;
                     
                        //extra validation for rules
                        if(!extraValidationRules($na, $matches_arr, $rule, $person_ref, $input)){
                            continue;
                        }  
                      
                        $found_match = true;
                        if(!array_key_exists($rule, $matches_arr)){
                            $matches_arr[$rule] = [];
                        }

                        $matches_arr[$rule][$input][$person_ref]  = $na_indexes;
                    }
                }
            }
            /**
             * 2b - deal with initials of this rule - skip if we already found match or we chose to avoid_initials
             */
          
            $initials_tables = $rule_data["initials_tables"];
            
            if(!empty($matching_str_initial)){
                foreach($initials_tables as $init_table){
                    $init_table_sql = $rule."_initial_".$init_table;

                    $sql = "SELECT DISTINCT input, GROUP_CONCAT(DISTINCT person_ref) AS person_ref,GROUP_CONCAT(DISTINCT na_index) AS na_indexes,
                                $init_table FROM full_analysis WHERE person_ref IN(SELECT DISTINCT person_ref FROM
                                 $init_table_sql WHERE rule_key IN (\"".$matching_str_initial."\")) group by input";
                    $res = sqlQuery($wl_mysql, $sql);
                    while($row = $res->fetch_object()){  
                        if(!empty($row->input)){
                            $person_ref = $row->person_ref;
                            $na_indexes = $row->na_indexes;       
                            $input = $row->input;
                            
                            if(!extraValidationRules($na, $matches_arr, $rule, $person_ref, $input)){
                                continue;
                            } 
                            
                            $initial_str = getCaptionFromFullAnalysis($row->$init_table);
                            $na_str = getCaptionFromFullAnalysis($na[$init_table] ?? null);
                            if(!validateInitialRule($initial_str,$na_str)){
                                continue;
                            }
                  
                            if(!array_key_exists($rule, $matches_arr)){
                                $matches_arr[$rule] = [];
                            }
                            if(!array_key_exists($input, $matches_arr[$rule])){
                                $matches_arr[$rule][$input] =  [];
                            }
                            $matches_arr[$rule][$input][$person_ref]  = $na_indexes;
                        }
                        
                    }
                }  
            }
        }      
    }

    /**
     * validate matches_arr
     */
    $matches_arr_final = validateMatches($matches_arr, $wl_mysql, $person_data["Name Alternatives"], $rule_arr);
    
    $res = $matches_arr_final["final_matches"];
    $results[] = ["ID" => $id, "Name" => $searchedPerson, "Rules" => $res, "matches_metadata"=> $matches_arr_final["metadata"]];
    return $results;
}

/**
 * check if at least one str is  a initial, otherwise return false
 * @param unknown $initial_str
 * @param unknown $na_str
 */
function validateInitialRule($initial_str,$na_str){
    if(strlen($initial_str)  !== 1 && strlen($na_str) !== 1){
        return false;
    }
    return true;
}
/**
 * getMatchingKeys - get the matchings keys for the sql search of the matches
 * @param unknown $wl_mysql
 * @param unknown $na
 * @param unknown $rule_data
 * @param unknown $keys_arr
 * @return unknown[]
 */
function getMatchingKeys($wl_mysql, $na, $rule_data, $keys_arr){
    $matching_keys_arr = $matching_keys_initial_arr = [];
    $mandatory_cond = true;
    
    $mapped_cols = $rule_data["mapped_cols"] ?? [];
    foreach($keys_arr as $key_str){
        
        $matching_key = $matching_k_init = "";
        $matching_key_arr = [];
        $mandatory_values_cond = true;
         $keys = explode('&', $key_str);
        foreach($keys as $k){
            
            $curr_data =  $na[$k] ?? "";
            $curr_key = getShortKeyVal($wl_mysql, $k, $curr_data);
            if(empty(($curr_key))){
                $mandatory_values_cond = false;
                $matching_key_arr = "";
                break;
            }
//             $matching_key .= $curr_key;
            $matching_key_arr[$k] = $curr_key;
        }
        
        if($mandatory_values_cond){
            ksort($matching_key_arr);
            $matching_key = join('',array_values($matching_key_arr));
            $matching_key = prepareMatchingKeyForSql($wl_mysql, $matching_key);
            $matching_keys_arr[] = $matching_key;
        }
    }
   
    //get keys for initials
    foreach($rule_data["initials_tables"] as $init_mand_f){
        $mandatory_values_cond = true;
        
        foreach($keys_arr as $key_str){
            $keys = explode('&', $key_str);
            foreach($keys as $k){
                $val = "";
                if(array_key_exists($k, $na)){            
                    $val = $na[$k]; 
                }
           
                $curr_data =  ($init_mand_f == $k) ? $val['caption'][0]  ?? $val[0] ?? "" : $val ?? "";
                $curr_key = getShortKeyVal($wl_mysql, $k, $curr_data);
                if(empty(($curr_key))){
                    $mandatory_values_cond = false;
                    break;
                }
                $matching_k_init .= $curr_key;
            }
            if($mandatory_values_cond){
                $matching_k_init = prepareMatchingKeyForSql($wl_mysql, $matching_k_init);
                $matching_keys_initial_arr[] =  $wl_mysql->real_escape_string($matching_k_init);
            }
        }
    }
    if(empty($matching_keys_arr) && empty($matching_keys_initial_arr)){
        $mandatory_cond = false;
    }
    
    return ["matching_keys_arr" => $matching_keys_arr, "matching_keys_initial_arr" => $matching_keys_initial_arr,
        "mandatory_values_cond" => $mandatory_cond
    ];
}


/**
 *
 * @param unknown $wl_mysql
 * @param unknown $na
 * @return number[]|unknown[]|NULL[]
 */
function getExactMatchData($wl_mysql, $na){
    $matches_arr= [];
    $caption =  $wl_mysql->real_escape_string($na["caption"]);
    $sql = "SELECT  person_ref, input  FROM full_analysis WHERE caption = '".$caption."' GROUP BY person_ref";
    $res = sqlQuery($wl_mysql, $sql);
    while ($row = $res->fetch_object()){
        $input = $row->input;
        $person_id = $row->person_ref;
        $matches_arr["possible_same_person"][$row->input][] = $person_id;// = ["ID"=>$person_id, "Name"=>$row->input, "score"=>100];
    }
    return $matches_arr;
}


function convertValForKey($wl_mysql, $val_arr){
    $val_str = "";
    if(!is_array($val_arr)){
        $val_arr = [$val_arr];
    }
    foreach($val_arr as $val){
        $val = $wl_mysql->real_escape_string($val);
        $arr = explode('(', $val);
        $val = trim($arr[0]);
        $val_str .= $val;
    }
    return $val_str;
}
/**
 * 
 * @param unknown $wl_mysql
 * @param unknown $field
 * @param unknown $val
 * @return string
 */
function getShortKeyVal($wl_mysql, $field, $val){
    if(is_array($val)){
        $val = $val["instance_arr"];
    }
    if(!is_array($val) && empty($val)){
        return null;
    }
   
    $val = convertValForKey($wl_mysql, $val); //mysql escape chars & remove () in value
    
    $arr = explode('_', $field);
    $key = "|";
    foreach($arr as $part){
       $key .= $part[0];
    }
    return "$key:$val|";
}

/**
 * getRuleFields - gets rules manadtory & optional fields 
 * @param unknown $rule_arr
 * @param unknown $rule
 * @return NULL[]|unknown[]
 */
function getRuleFields($rule_arr, $rule){
    if(strpos($rule, '_initial_')){
        $temp_arr = explode('_initial_', $rule);
        $rule_str = $temp_arr[0];
    }
    else{
        $rule_str = $rule;
    }
    
    $optional_arr = $rule_arr[$rule_str]["optional"] ?? null;
    
    $mandatory_arr = $rule_arr[$rule_str]["mandatory"] ?? null;
    
    return ["optional" => $optional_arr, "mandatory" => $mandatory_arr];
}



/**
 *
 * @param unknown $wl_mysql
 * @param unknown $ivEntityMatcherService
 * @param unknown $wlSelected
 * @param unknown $entities_data
 * @return number[]|unknown[]|array[][]|unknown[][]
 */
function searchList($wl_mysql, $ivEntityMatcherService, $wlSelected, $rules_json, $columns_metadata, $entities_data, $reverseOrder){
    $entities_data = explode("\n", base64_decode($entities_data));
    $id = $index = 0;
    $total = count($entities_data);
    write_to_log("INFO", "Running $index / $total");
    
    
    foreach($entities_data as $searchedPerson){
  
        $index++;
       
        $searched_person_data = fetchPersonData($ivEntityMatcherService, $searchedPerson, $reverseOrder);
        $resEM = searchPerson($wl_mysql, $searchedPerson, $searched_person_data, $wlSelected, $rules_json, $columns_metadata, $id);
        if(!empty($resEM)){
            $results[] = $resEM[0];
        }
        $id++;
        /* do stuff here */
        write_to_log("INFO", "Running $index / $total");
        
    }
    return $results;
}

/**
 * Extra checks patch
 * @param unknown $cur_res
 * @param unknown $curr_id
 * @param unknown $extra_input
 */
function extraChecks(&$cur_res, $curr_id = null, $extra_input = null){
    /**
     * for csv only - checking input of middle name for arab names (from output)
     */
    //extra arab middle name check
    $mid_arab_check = true;
    if(!empty($extra_input)){
        $curret_extra_input = array_key_exists($curr_id, $extra_input) ? $extra_input[$curr_id] : $extra_input;
        foreach ($curret_extra_input as $key=>$value){
            $key_clean = strtolower(str_replace('_', ' ', $key));
            
            //check if we fetch arab name and the input of middle name was initials
            if(!empty($value) && strlen($value) == 1 && strpos($key_clean, 'middle name') !== false && $cur_res["ethnicity"] == 'arab' && $mid_arab_check){
                $mid_arab_check = false;
                if(!empty($cur_res["warning"] )){
                    $cur_res["warning"] .= "& Arab names do not use middle name";
                }
                else{
                    $cur_res["warning"] = "Arab names do not use middle name";
                }
            }
            $cur_res[$key] = $value;
        }
    }
    
    /**
     * check initials in the given, middle and family names
     */
    
    //if Chinese - ignore extra checks
    if($cur_res["naming_convention"] == "Chinese" || stripos($cur_res["ethnicity"], "Chinese") !== false ||
        $cur_res["naming_convention"] == "Korean" || stripos($cur_res["ethnicity"], "Korean") !== false){
            return;
    }
    if(initials_check($cur_res["given_name"])){
        if(!empty($cur_res["warning"] )){
            $cur_res["warning"] .= "& ";
        }
        
        if($cur_res["naming_convention"] == "Indian"){
            $cur_res["warning"] =  "This seems to be an Indian or Hindi name - the initials could signify different roles according to origin - father's first name, clan etc";
        }
        else{
            $cur_res["warning"] = "The given name contains only 1 or 2 letters - it could be truncated or an initial or short form for a name prefix (A for Abed, B for Bin, etc.)";
        }
    }
    
    if(initials_check($cur_res["middle_name"]) || initials_check($cur_res["middle_name_second"]) ||
        initials_check($cur_res["middle_name_third"])){
            if(!empty($cur_res["warning"] )){
                $cur_res["warning"] .= "& ";
            }
            $cur_res["warning"] = "The  name contains an element with only (1 or 2) letters - it could be truncated, short form for a name prefix (V for Van, A for Abed, B for Bin etc.)  or an initial";
    }
    
    if(initials_check($cur_res["family_name"])){
        if(!empty($cur_res["Warning"] )){
            $cur_res["warning"] .= "& ";
        }
        $cur_res["warning"] = "The name seems to be an initial of a Family Name";
    }
}

function saveToCsv($jsonDecoded, $name){
    $date = date('m_d_Y_h_i_s', time());
    
    //Give our CSV file a name.
    $csvFileName = '.'.DIRECTORY_SEPARATOR.'tmp'.DIRECTORY_SEPARATOR.$name.'_'.$date.".csv";
    
    //Open file pointer.
    $fp = fopen($csvFileName, 'w');
    
    //Loop through the associative array.
    foreach($jsonDecoded as $row){
        //Write the row to the CSV file.
        fputcsv($fp, $row);
    }
    
    //Finally, close the file pointer.
    fclose($fp);
}

function array_iunique( $array ) {
    return array_intersect_key(
        $array,
        array_unique( array_map( "strtolower", $array ) )
        );
}

function isStringJson($string) {
    json_decode($string);
    return (json_last_error() == JSON_ERROR_NONE);
}

/**
 * initials checkup
 * x, xy, x.,  x.y.,
 */
function initials_check($name){
    if(empty($name)){
        return false;
    }
    //if name is x, xy, x., xy.
    if(strlen($name) < 3){
        return true;
    }
    //if name is x.y.
    elseif(strlen($name) == 4 && strpos($name, '.') !== false){
        return true;
    }
    return false;
}
/**
 * creates orthography string from parts in the results
 * @param unknown $orthography
 * @return string
 */
function orginizeOrthography($orthography){
    $res = "";
    $order_orth_parts = ["given_name",
        "father_patrimonyal_name",
        "grandfather_patrimonyal_name",
        "great_grandfather_patronymic_name",
        "great_great_grandfather_patronymic_name",
        "family_name",
        "kunya",
        "tribal_name",
        "nisba"];
    
    foreach($order_orth_parts as $part){
        if(!empty($orthography[$part])){
            $res .= "$orthography[$part] ";
        }
    }
    return $res;
    
}

/**
 * read csv file
 * @param unknown $file_name
 * @return array
 */
function readCsvFile($file_name){
    $row = 1;
    if (($handle = fopen($file_name, "r")) !== FALSE) {
        while (($data = fgetcsv($handle, 1000, ";")) !== FALSE) {
            $res[] =  $data;
        }
        fclose($handle);
    }
    return $res;
}

/**
 * getWLName
 * @param unknown $list
 * @return unknown|boolean
 */
function getWLName($list){
    if(!empty($list['fileData']['name'])){
        return $list['fileData']['name'];
    }
    return false;
}

 /**
 * getListChunks
 * @param unknown $names_arr
 * @return number[]|string[][]
 */
function getListChunks($names_arr, $bulk_limit){
    $file_Data_arr = [];
    $totalNames = count($names_arr) - 1;
    $names_chunks = array_chunk($names_arr, $bulk_limit);
    foreach($names_chunks as $chunk){
        $names_str_chunk = implode("\n", $chunk);
        $file_Data_arr[] = base64_encode($names_str_chunk);
    }
    
    return ["total"=> $totalNames, "file_Data_arr"=>$file_Data_arr];
}

function validatedCsvLine($line, $csv_seperator, $curr_id){
    $validUTF8 = ! (false === mb_detect_encoding($line, 'UTF-8', true));
    if(!$validUTF8){
        return ["Error" => "invalid utf8 line $curr_id"];
    }
    $seperator_bool = strpos($line, $csv_seperator);
    if($seperator_bool  === false){
        return ["Error" => "Line $curr_id doesnt contain Seperator $csv_seperator"];
    }
    
    return true;
    
}
/**
 * 
 * @param unknown $name
 * @param unknown $csv_seperator
 * @param unknown $first
 * @return void|NULL|NULL[][]|unknown[][]|unknown[]|string[]
 */
function handleCSV($line, $csv_seperator, &$first, &$input_fields, &$name_index, $curr_id){
    $extra_input = [];
    $valid_res = validatedCsvLine($line, $csv_seperator, $curr_id);
    if(is_array($valid_res) && array_key_exists("Error", $valid_res)){
        write_to_log("ERROR", "Handle Csv error: $valid_res[Error]");
        die(json_encode($valid_res, true));
        return $valid_res;
    }
    
    $data = str_getcsv($line, $csv_seperator);
    if(empty($data) || empty($data[0])){
        return ;
    }
    //configure the columns
    if($first){
        $first = false;
        $name_col_id = array_search("name", $data);  //look for the "name" columns index
        //if not configure such column
        if(empty($name_col_id)){
            foreach($data as $f){
                $input_fields[] = $f;
            }
            foreach($data as $col){
                if(stripos($col, 'name') !== false){
                    $name_index[] = array_search($col, $data);
                }
            }
        }
        else{
            $name_index = $name_col_id;
            unset($data[$name_col_id]);
            $input_fields = $data;
            
        }
        
        return null;
    }
    
    if(!empty($data)){
        foreach ($input_fields as $key=>$field){
            $field = strtolower($field);
            $extra_input["input_$field"] = $data[$key] ?? null;
        }
    }
    if(empty($name_index) || is_array($name_index) && !empty($data)){
        
        foreach($name_index as $nc_index){
            $name_vals[] = $data[$nc_index] ?? null;
        }
        
        $count = -1;
        
        $name = implode(' ', $name_vals);
    }
    else{
        $name = $data[$name_index];
    }
    
    return array("name"=>$name, "extra_input"=>$extra_input, "name_index"=> $name_index);
}



/**
 * analyseName - analyse a specific name and return the relevant columns
 * @param unknown $searchedPerson
 * @return string[][]
 */
function analyseName($searchedPerson,$ivEntityMatcherService,  $columns_metadata, $resultsCols, $reverseOrder){
    write_to_log("TRACE", "Analysing the name: $searchedPerson");
    $results = array();
    $allowed_cols = [];
    $actionElements = fetchPersonData($ivEntityMatcherService, $searchedPerson,  $reverseOrder);
    
    $eth = $actionElements["ethnicity"] ?? null;
    
    /**
     * A - if we have analysis
     */
    if(!empty($actionElements["Name Alternatives"])){
        $index = 0;
        foreach($actionElements["Name Alternatives"] as $na){
            $cur_res = array_combine($resultsCols, array_fill(0, count($resultsCols), ''));
            
            $index_str = (count($actionElements["Name Alternatives"]) > 1) ? $index_str = ' ('.++$index.')' : "";
            
            $cur_res["input"] = $searchedPerson.$index_str;
            
            
            foreach($cur_res as $col=>$val){
                if(array_key_exists($col, $na)){
                    if(is_array($na[$col])){
                        $na[$col] = !empty($_REQUEST["instances"]) ? $na[$col]["instance"] : $na[$col]["caption"];
                    }
                    //A- checking for orthography names
                    if(strpos(strtolower($col), "name") !== false && strpos(strtolower($col), "chinese") === false && strpos(($na[$col]), '(') !== false){
                        $name_exp = explode('(', $na[$col]);
                        if(!array_key_exists("orthography", $cur_res) || is_string($cur_res["orthography"])){
                            $cur_res["orthography"] = [];
                        }
                        $k = $cur_res["orthography"];
                        $cur_res["orthography"][$col] =  rtrim($name_exp[1],')');
                        $cur_res[$col] = $name_exp[0];
                        continue;
                    }
                    
                    //B - check for mapped columns
                    if(array_key_exists($col, $columns_metadata["mappedColsView"])){
                        $new_col = $columns_metadata["mappedColsView"][$col];
                        
                        if(!empty($cur_res[$new_col])){
                            $cur_res[$new_col] .= $na[$col];
                        }
                        else{
                            $cur_res[$new_col] = $na[$col];
                        }
                        continue;
                    }
                    //C - regular
                    $cur_res[$col] = $na[$col];
                }
            }
            
            if((!array_key_exists('ethnicity', $cur_res) || empty($cur_res['ethnicity'])) && !empty($eth)){
                $cur_res['ethnicity'] = $eth;
            }
            
            extraChecks($cur_res);
            
            $allowed_cols = array_unique(array_merge($allowed_cols, array_keys(array_filter(((array)$cur_res)))));
            
            //if orthography are defined - convert it to string by order
            if(!empty($cur_res["orthography"])){
                $cur_res["orthography"] = orginizeOrthography($cur_res["orthography"]);
            }
            $results[] = $cur_res;
        }
    }
    
    /**
     * B - no analysis (Error)
     */
    if(!empty($actionElements["error"])){
        $cur_res["input"] = $searchedPerson;
        $cur_res["warning"] = $actionElements["error"];
        $allowed_cols = array_unique(array_merge($allowed_cols, array_keys(array_filter(((array)$cur_res)))));
        $results[] = $cur_res;
    }
    
    $final_res = [];
    foreach($results as $element){
        $temp_element = [];
        foreach ($element as $key => $value)
        {
            
            //place the value if its allowed col
            if (in_array($key, $allowed_cols)){
                $temp_element[$key] = $value;
            }
        }
        $final_res[] = $temp_element;
    }
    return $final_res;
}






