Optimize DXCC list queries and add DB indexes

Refactored Workabledxcc controller and model to batch DXCC entity and worked/confirmed status lookups for improved performance. Added migration to create composite indexes on DXCC-related columns to further speed up queries. Updated migration version to 205.

application/config/migration.php

@@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE;
 |
 */
 
-$config['migration_version'] = 204;
+$config['migration_version'] = 205;
 
 /*
 |--------------------------------------------------------------------------

application/controllers/Workabledxcc.php

@@ -29,53 +29,47 @@ class Workabledxcc extends CI_Controller
 
 	public function dxcclist()
 	{
-
 		$json = file_get_contents($this->optionslib->get_option('dxped_url'));
-
-		// Decode the JSON data into a PHP array
 		$dataResult = json_decode($json, true);
 
-		// Initialize an empty array to store the required data
+		if (empty($dataResult)) {
-		$requiredData = array();
+			$data['dxcclist'] = array();
+			$this->load->view('/workabledxcc/components/dxcclist', $data);
+			return;
+		}
 
 		// Get Date format
 		if ($this->session->userdata('user_date_format')) {
-			// If Logged in and session exists
 			$custom_date_format = $this->session->userdata('user_date_format');
 		} else {
-			// Get Default date format from /config/cloudlog.php
 			$custom_date_format = $this->config->item('qso_date_format');
 		}
 
-		// Iterate through the decoded JSON data
+		// Load models once
-		foreach ($dataResult as $item) {
+		$this->load->model('logbook_model');
-			// Create a new array with the required fields and add it to the main array
+		$this->load->model('Workabledxcc_model');
-			$oldStartDate = DateTime::createFromFormat('Y-m-d', $item['0']);
 
+		// Get all DXCC entities for all callsigns in one batch
+		$callsigns = array_column($dataResult, 'callsign');
+		$dates = array_column($dataResult, '0');
+		$dxccEntities = $this->Workabledxcc_model->batchDxccLookup($callsigns, $dates);
+
+		// Get worked/confirmed status for all entities in batch
+		$uniqueEntities = array_unique(array_filter($dxccEntities));
+		$dxccStatus = $this->Workabledxcc_model->batchDxccWorkedStatus($uniqueEntities);
+
+		// Process results
+		$requiredData = array();
+		foreach ($dataResult as $index => $item) {
+			$oldStartDate = DateTime::createFromFormat('Y-m-d', $item['0']);
 			$StartDate = $oldStartDate->format($custom_date_format);
 
 			$oldEndDate = DateTime::createFromFormat('Y-m-d', $item['1']);
-
 			$EndDate = $oldEndDate->format($custom_date_format);
 
-			$oldStartDate1 = DateTime::createFromFormat('Y-m-d', $item['0']);
+			// Get DXCC status for this callsign
-
+			$entity = $dxccEntities[$index] ?? null;
-			$StartDate1 = $oldStartDate1->format('Y-m-d');
+			$worked = $entity && isset($dxccStatus[$entity]) ? $dxccStatus[$entity] : ['workedBefore' => false, 'confirmed' => false];
-
-
-			$this->load->model('logbook_model');
-			$dxccInfo = $this->logbook_model->dxcc_lookup($item['callsign'], $StartDate1);
-
-			// Call DXCC Worked function to check if the DXCC has been worked before
-			if (isset($dxccInfo['entity'])) {
-				$dxccWorked = $this->dxccWorked($dxccInfo['entity']);
-			} else {
-				// Handle the case where 'entity' is not set in $dxccInfo
-				$dxccWorked = array(
-					'workedBefore' => false,
-					'confirmed' => false,
-				);
-			}
 
 			$requiredData[] = array(
 				'clean_date' => $item['0'],
@@ -84,15 +78,12 @@ class Workabledxcc extends CI_Controller
 				'country' => $item['2'],
 				'notes' => $item['6'],
 				'callsign' => $item['callsign'],
-				'workedBefore' => $dxccWorked['workedBefore'],
+				'workedBefore' => $worked['workedBefore'],
-				'confirmed' => $dxccWorked['confirmed'],
+				'confirmed' => $worked['confirmed'],
 			);
 		}
 
 		$data['dxcclist'] = $requiredData;
-
-		// Return the array with the required data
-
 		$this->load->view('/workabledxcc/components/dxcclist', $data);
 	}
 

application/migrations/205_add_workable_dxcc_indexes.php

@@ -0,0 +1,49 @@
+<?php
+
+defined('BASEPATH') OR exit('No direct script access allowed');
+
+class Migration_add_workable_dxcc_indexes extends CI_Migration {
+
+    public function up()
+    {
+        // Add composite index for workable DXCC queries
+        // This will greatly improve performance for COL_COUNTRY + station_id + COL_PROP_MODE queries
+        $this->db->db_debug = false;
+        
+        // Check if index already exists
+        $index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_workable_dxcc'")->num_rows();
+        
+        if ($index_exists == 0) {
+            $sql = "ALTER TABLE ".$this->config->item('table_name')." ADD INDEX `idx_workable_dxcc` (`COL_COUNTRY`, `station_id`, `COL_PROP_MODE`)";
+            $this->db->query($sql);
+        }
+        
+        // Add index for confirmation status columns
+        $conf_index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_qsl_confirmations'")->num_rows();
+        
+        if ($conf_index_exists == 0) {
+            $sql = "ALTER TABLE ".$this->config->item('table_name')." ADD INDEX `idx_qsl_confirmations` (`COL_QSL_RCVD`, `COL_LOTW_QSL_RCVD`, `COL_EQSL_QSL_RCVD`, `COL_QRZCOM_QSO_DOWNLOAD_STATUS`)";
+            $this->db->query($sql);
+        }
+        
+        $this->db->db_debug = true;
+    }
+
+    public function down()
+    {
+        $this->db->db_debug = false;
+        
+        // Drop the indexes if they exist
+        $index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_workable_dxcc'")->num_rows();
+        if ($index_exists > 0) {
+            $this->db->query("ALTER TABLE ".$this->config->item('table_name')." DROP INDEX `idx_workable_dxcc`");
+        }
+        
+        $conf_index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_qsl_confirmations'")->num_rows();
+        if ($conf_index_exists > 0) {
+            $this->db->query("ALTER TABLE ".$this->config->item('table_name')." DROP INDEX `idx_qsl_confirmations`");
+        }
+        
+        $this->db->db_debug = true;
+    }
+}

application/models/Workabledxcc_model.php

@@ -2,87 +2,221 @@
 
 class Workabledxcc_model extends CI_Model
 {
+    // Cache for DXCC lookups to avoid repeated queries
+    private $dxccCache = array();
+    private $workedCache = array();
+
+    /**
+     * Batch DXCC lookup for multiple callsigns
+     * @param array $callsigns Array of callsigns
+     * @param array $dates Array of dates corresponding to callsigns
+     * @return array Array of DXCC entities indexed by callsign index
+     */
+    public function batchDxccLookup($callsigns, $dates)
+    {
+        $this->load->model('logbook_model');
+        $entities = array();
+        
+        foreach ($callsigns as $index => $callsign) {
+            $cacheKey = $callsign . '_' . $dates[$index];
+            
+            if (!isset($this->dxccCache[$cacheKey])) {
+                $dxccInfo = $this->logbook_model->dxcc_lookup($callsign, $dates[$index]);
+                $this->dxccCache[$cacheKey] = isset($dxccInfo['entity']) ? $dxccInfo['entity'] : null;
+            }
+            
+            $entities[$index] = $this->dxccCache[$cacheKey];
+        }
+        
+        return $entities;
+    }
+
+    /**
+     * Batch check if DXCC entities have been worked/confirmed
+     * @param array $entities Array of unique DXCC entities
+     * @return array Array of worked/confirmed status indexed by entity
+     */
+    public function batchDxccWorkedStatus($entities)
+    {
+        if (empty($entities)) {
+            return array();
+        }
+
+        $user_default_confirmation = $this->session->userdata('user_default_confirmation');
+        $this->load->model('logbooks_model');
+        $logbooks_locations_array = $this->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook'));
+        
+        if (empty($logbooks_locations_array)) {
+            return array_fill_keys($entities, ['workedBefore' => false, 'confirmed' => false]);
+        }
+
+        $results = array();
+        
+        // Build confirmation criteria once
+        $confirmationCriteria = $this->buildConfirmationCriteria($user_default_confirmation);
+        
+        // Batch query for worked status
+        $workedResults = $this->batchWorkedQuery($entities, $logbooks_locations_array);
+        
+        // Batch query for confirmed status  
+        $confirmedResults = $this->batchConfirmedQuery($entities, $logbooks_locations_array, $confirmationCriteria);
+        
+        // Combine results
+        foreach ($entities as $entity) {
+            $results[$entity] = [
+                'workedBefore' => isset($workedResults[$entity]),
+                'confirmed' => isset($confirmedResults[$entity])
+            ];
+        }
+        
+        return $results;
+    }
+
+    /**
+     * Build confirmation criteria SQL based on user preferences
+     */
+    private function buildConfirmationCriteria($user_default_confirmation)
+    {
+        $criteria = array();
+        
+        if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'Q') !== false) {
+            $criteria[] = "COL_QSL_RCVD='Y'";
+        }
+        if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'L') !== false) {
+            $criteria[] = "COL_LOTW_QSL_RCVD='Y'";
+        }
+        if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'E') !== false) {
+            $criteria[] = "COL_EQSL_QSL_RCVD='Y'";
+        }
+        if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'Z') !== false) {
+            $criteria[] = "COL_QRZCOM_QSO_DOWNLOAD_STATUS='Y'";
+        }
+        
+        return empty($criteria) ? '1=0' : '(' . implode(' OR ', $criteria) . ')';
+    }
+
+    /**
+     * Batch query to check which entities have been worked
+     */
+    private function batchWorkedQuery($entities, $logbooks_locations_array)
+    {
+        // Use a single query with GROUP BY to check all entities at once
+        $this->db->select('COL_COUNTRY')
+                 ->distinct()
+                 ->from($this->config->item('table_name'))
+                 ->where('COL_PROP_MODE !=', 'SAT')
+                 ->where_in('station_id', $logbooks_locations_array)
+                 ->where_in('COL_COUNTRY', array_map('urlencode', $entities));
+        
+        $query = $this->db->get();
+        $results = array();
+        
+        foreach ($query->result() as $row) {
+            $results[urldecode($row->COL_COUNTRY)] = true;
+        }
+        
+        return $results;
+    }
+
+    /**
+     * Batch query to check which entities have been confirmed
+     */
+    private function batchConfirmedQuery($entities, $logbooks_locations_array, $confirmationCriteria)
+    {
+        if ($confirmationCriteria === '1=0') {
+            return array();
+        }
+        
+        $this->db->select('COL_COUNTRY')
+                 ->distinct()
+                 ->from($this->config->item('table_name'))
+                 ->where('COL_PROP_MODE !=', 'SAT')
+                 ->where($confirmationCriteria)
+                 ->where_in('station_id', $logbooks_locations_array)
+                 ->where_in('COL_COUNTRY', array_map('urlencode', $entities));
+        
+        $query = $this->db->get();
+        $results = array();
+        
+        foreach ($query->result() as $row) {
+            $results[urldecode($row->COL_COUNTRY)] = true;
+        }
+        
+        return $results;
+    }
 
     public function GetThisWeek()
     {
         $json = file_get_contents($this->optionslib->get_option('dxped_url'));
-
-        // Step 2: Convert the JSON data to an array.
         $data = json_decode($json, true);
 
-        // Step 3: Create a new array to hold the records for this week.
+        if (empty($data)) {
-        $thisWeekRecords = [];
+            return array();
+        }
 
-        // Get the start and end of this week.
+        $thisWeekRecords = [];
         $startOfWeek = (new DateTime())->setISODate((new DateTime())->format('o'), (new DateTime())->format('W'), 1);
         $endOfWeek = (clone $startOfWeek)->modify('+6 days');
 
-        // Step 4: Iterate over the array.
+        // Get Date format
-        foreach ($data as $record) {
+        if ($this->session->userdata('user_date_format')) {
+            $custom_date_format = $this->session->userdata('user_date_format');
+        } else {
+            $custom_date_format = $this->config->item('qso_date_format');
+        }
 
-            // Convert "0" and "1" to DateTime objects.
+        // First pass: filter records for this week
+        $weekRecords = array();
+        foreach ($data as $record) {
             $startDate = new DateTime($record['0']);
             $endDate = new DateTime($record['1']);
 
-            // Step 5: Check if the start date or end date is within this week.
+            if (($startDate >= $startOfWeek && $startDate <= $endOfWeek) || 
-            if (($startDate >= $startOfWeek && $startDate <= $endOfWeek) || ($endDate >= $startOfWeek && $endDate <= $endOfWeek)) {
+                ($endDate >= $startOfWeek && $endDate <= $endOfWeek)) {
+                $weekRecords[] = $record;
+            }
+        }
+
+        if (empty($weekRecords)) {
+            return array();
+        }
+
+        // Batch process DXCC lookups
+        $callsigns = array_column($weekRecords, 'callsign');
+        $dates = array_column($weekRecords, '0');
+        $dxccEntities = $this->batchDxccLookup($callsigns, $dates);
+
+        // Get worked/confirmed status for all entities in batch
+        $uniqueEntities = array_unique(array_filter($dxccEntities));
+        $dxccStatus = $this->batchDxccWorkedStatus($uniqueEntities);
+
+        // Process results
+        foreach ($weekRecords as $index => $record) {
             $endDate = new DateTime($record['1']);
             $now = new DateTime();
             $interval = $now->diff($endDate);
             $daysLeft = $interval->days;
 
-                // If daysLeft is 0, set it to "Last day"
+            $daysLeft = ($daysLeft == 0) ? "Last day" : $daysLeft . " days left";
-                if ($daysLeft == 0) {
-                    $daysLeft = "Last day";
-                } else {
-                    $daysLeft = $daysLeft . " days left";
-                }
-
-                // Add daysLeft to record
             $record['daysLeft'] = $daysLeft;
-                // Get Date format
-                if ($this->session->userdata('user_date_format')) {
-                    // If Logged in and session exists
-                    $custom_date_format = $this->session->userdata('user_date_format');
-                } else {
-                    // Get Default date format from /config/cloudlog.php
-                    $custom_date_format = $this->config->item('qso_date_format');
-                }
 
-                // Create a new array with the required fields and add it to the main array
             $oldStartDate = DateTime::createFromFormat('Y-m-d', $record['0']);
-
+            $record['startDate'] = $oldStartDate->format($custom_date_format);
-                $StartDate = $oldStartDate->format($custom_date_format);
-                $record['startDate'] = $StartDate;
 
             $oldEndDate = DateTime::createFromFormat('Y-m-d', $record['1']);
-                $EndDate = $oldEndDate->format($custom_date_format);
+            $record['endDate'] = $oldEndDate->format($custom_date_format);
-                $record['endDate'] = $EndDate;
 
-                $record['confirmed'] = true; // or false, depending on your logic
+            // Get DXCC status for this callsign
+            $entity = $dxccEntities[$index] ?? null;
+            $worked = $entity && isset($dxccStatus[$entity]) ? $dxccStatus[$entity] : ['workedBefore' => false, 'confirmed' => false];
 
-                $CI = &get_instance();
+            $record['workedBefore'] = $worked['workedBefore'];
-                $CI->load->model('logbook_model');
+            $record['confirmed'] = $worked['confirmed'];
-                $dxccInfo = $CI->logbook_model->dxcc_lookup($record['callsign'], $startDate->format('Y-m-d'));
-
-                // Call DXCC Worked function to check if the DXCC has been worked before
-                if (isset($dxccInfo['entity'])) {
-                    $dxccWorkedData = $this->dxccWorked($dxccInfo['entity']);
-                    $record = array_merge($record, $dxccWorkedData);
-                } else {
-                    // Handle the case where 'entity' is not set in $dxccInfo
-                    $itemsToAdd = array(
-                        'workedBefore' => false,
-                        'confirmed' => false,
-                    );
-                    $record = array_merge($record, $itemsToAdd);
-                }
 
             $thisWeekRecords[] = $record;
         }
-        }
-        return $thisWeekRecords;
 
+        return $thisWeekRecords;
     }
 
     function dxccWorked($country)