From 9b17ff9250b281a255c20a0da49416f359b65873 Mon Sep 17 00:00:00 2001 From: Peter Goodhall Date: Mon, 11 Aug 2025 14:50:56 +0100 Subject: [PATCH] Optimize DXCC list queries and add DB indexes Refactored Workabledxcc controller and model to batch DXCC entity and worked/confirmed status lookups for improved performance. Added migration to create composite indexes on DXCC-related columns to further speed up queries. Updated migration version to 205. --- application/config/migration.php | 2 +- application/controllers/Workabledxcc.php | 61 ++-- .../205_add_workable_dxcc_indexes.php | 49 ++++ application/models/Workabledxcc_model.php | 264 +++++++++++++----- 4 files changed, 275 insertions(+), 101 deletions(-) create mode 100644 application/migrations/205_add_workable_dxcc_indexes.php diff --git a/application/config/migration.php b/application/config/migration.php index 60928918..1b160613 100644 --- a/application/config/migration.php +++ b/application/config/migration.php @@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE; | */ -$config['migration_version'] = 204; +$config['migration_version'] = 205; /* |-------------------------------------------------------------------------- diff --git a/application/controllers/Workabledxcc.php b/application/controllers/Workabledxcc.php index e28ed67b..3654004c 100644 --- a/application/controllers/Workabledxcc.php +++ b/application/controllers/Workabledxcc.php @@ -29,53 +29,47 @@ class Workabledxcc extends CI_Controller public function dxcclist() { - $json = file_get_contents($this->optionslib->get_option('dxped_url')); - - // Decode the JSON data into a PHP array $dataResult = json_decode($json, true); - // Initialize an empty array to store the required data - $requiredData = array(); + if (empty($dataResult)) { + $data['dxcclist'] = array(); + $this->load->view('/workabledxcc/components/dxcclist', $data); + return; + } // Get Date format if ($this->session->userdata('user_date_format')) { - // If Logged in and session exists $custom_date_format = $this->session->userdata('user_date_format'); } else { - // Get Default date format from /config/cloudlog.php $custom_date_format = $this->config->item('qso_date_format'); } - // Iterate through the decoded JSON data - foreach ($dataResult as $item) { - // Create a new array with the required fields and add it to the main array - $oldStartDate = DateTime::createFromFormat('Y-m-d', $item['0']); + // Load models once + $this->load->model('logbook_model'); + $this->load->model('Workabledxcc_model'); + // Get all DXCC entities for all callsigns in one batch + $callsigns = array_column($dataResult, 'callsign'); + $dates = array_column($dataResult, '0'); + $dxccEntities = $this->Workabledxcc_model->batchDxccLookup($callsigns, $dates); + + // Get worked/confirmed status for all entities in batch + $uniqueEntities = array_unique(array_filter($dxccEntities)); + $dxccStatus = $this->Workabledxcc_model->batchDxccWorkedStatus($uniqueEntities); + + // Process results + $requiredData = array(); + foreach ($dataResult as $index => $item) { + $oldStartDate = DateTime::createFromFormat('Y-m-d', $item['0']); $StartDate = $oldStartDate->format($custom_date_format); $oldEndDate = DateTime::createFromFormat('Y-m-d', $item['1']); - $EndDate = $oldEndDate->format($custom_date_format); - $oldStartDate1 = DateTime::createFromFormat('Y-m-d', $item['0']); - - $StartDate1 = $oldStartDate1->format('Y-m-d'); - - - $this->load->model('logbook_model'); - $dxccInfo = $this->logbook_model->dxcc_lookup($item['callsign'], $StartDate1); - - // Call DXCC Worked function to check if the DXCC has been worked before - if (isset($dxccInfo['entity'])) { - $dxccWorked = $this->dxccWorked($dxccInfo['entity']); - } else { - // Handle the case where 'entity' is not set in $dxccInfo - $dxccWorked = array( - 'workedBefore' => false, - 'confirmed' => false, - ); - } + // Get DXCC status for this callsign + $entity = $dxccEntities[$index] ?? null; + $worked = $entity && isset($dxccStatus[$entity]) ? $dxccStatus[$entity] : ['workedBefore' => false, 'confirmed' => false]; $requiredData[] = array( 'clean_date' => $item['0'], @@ -84,15 +78,12 @@ class Workabledxcc extends CI_Controller 'country' => $item['2'], 'notes' => $item['6'], 'callsign' => $item['callsign'], - 'workedBefore' => $dxccWorked['workedBefore'], - 'confirmed' => $dxccWorked['confirmed'], + 'workedBefore' => $worked['workedBefore'], + 'confirmed' => $worked['confirmed'], ); } $data['dxcclist'] = $requiredData; - - // Return the array with the required data - $this->load->view('/workabledxcc/components/dxcclist', $data); } diff --git a/application/migrations/205_add_workable_dxcc_indexes.php b/application/migrations/205_add_workable_dxcc_indexes.php new file mode 100644 index 00000000..3c8cd90c --- /dev/null +++ b/application/migrations/205_add_workable_dxcc_indexes.php @@ -0,0 +1,49 @@ +db->db_debug = false; + + // Check if index already exists + $index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_workable_dxcc'")->num_rows(); + + if ($index_exists == 0) { + $sql = "ALTER TABLE ".$this->config->item('table_name')." ADD INDEX `idx_workable_dxcc` (`COL_COUNTRY`, `station_id`, `COL_PROP_MODE`)"; + $this->db->query($sql); + } + + // Add index for confirmation status columns + $conf_index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_qsl_confirmations'")->num_rows(); + + if ($conf_index_exists == 0) { + $sql = "ALTER TABLE ".$this->config->item('table_name')." ADD INDEX `idx_qsl_confirmations` (`COL_QSL_RCVD`, `COL_LOTW_QSL_RCVD`, `COL_EQSL_QSL_RCVD`, `COL_QRZCOM_QSO_DOWNLOAD_STATUS`)"; + $this->db->query($sql); + } + + $this->db->db_debug = true; + } + + public function down() + { + $this->db->db_debug = false; + + // Drop the indexes if they exist + $index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_workable_dxcc'")->num_rows(); + if ($index_exists > 0) { + $this->db->query("ALTER TABLE ".$this->config->item('table_name')." DROP INDEX `idx_workable_dxcc`"); + } + + $conf_index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_qsl_confirmations'")->num_rows(); + if ($conf_index_exists > 0) { + $this->db->query("ALTER TABLE ".$this->config->item('table_name')." DROP INDEX `idx_qsl_confirmations`"); + } + + $this->db->db_debug = true; + } +} diff --git a/application/models/Workabledxcc_model.php b/application/models/Workabledxcc_model.php index d7a1860d..f4989dd0 100644 --- a/application/models/Workabledxcc_model.php +++ b/application/models/Workabledxcc_model.php @@ -2,87 +2,221 @@ class Workabledxcc_model extends CI_Model { + // Cache for DXCC lookups to avoid repeated queries + private $dxccCache = array(); + private $workedCache = array(); + + /** + * Batch DXCC lookup for multiple callsigns + * @param array $callsigns Array of callsigns + * @param array $dates Array of dates corresponding to callsigns + * @return array Array of DXCC entities indexed by callsign index + */ + public function batchDxccLookup($callsigns, $dates) + { + $this->load->model('logbook_model'); + $entities = array(); + + foreach ($callsigns as $index => $callsign) { + $cacheKey = $callsign . '_' . $dates[$index]; + + if (!isset($this->dxccCache[$cacheKey])) { + $dxccInfo = $this->logbook_model->dxcc_lookup($callsign, $dates[$index]); + $this->dxccCache[$cacheKey] = isset($dxccInfo['entity']) ? $dxccInfo['entity'] : null; + } + + $entities[$index] = $this->dxccCache[$cacheKey]; + } + + return $entities; + } + + /** + * Batch check if DXCC entities have been worked/confirmed + * @param array $entities Array of unique DXCC entities + * @return array Array of worked/confirmed status indexed by entity + */ + public function batchDxccWorkedStatus($entities) + { + if (empty($entities)) { + return array(); + } + + $user_default_confirmation = $this->session->userdata('user_default_confirmation'); + $this->load->model('logbooks_model'); + $logbooks_locations_array = $this->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook')); + + if (empty($logbooks_locations_array)) { + return array_fill_keys($entities, ['workedBefore' => false, 'confirmed' => false]); + } + + $results = array(); + + // Build confirmation criteria once + $confirmationCriteria = $this->buildConfirmationCriteria($user_default_confirmation); + + // Batch query for worked status + $workedResults = $this->batchWorkedQuery($entities, $logbooks_locations_array); + + // Batch query for confirmed status + $confirmedResults = $this->batchConfirmedQuery($entities, $logbooks_locations_array, $confirmationCriteria); + + // Combine results + foreach ($entities as $entity) { + $results[$entity] = [ + 'workedBefore' => isset($workedResults[$entity]), + 'confirmed' => isset($confirmedResults[$entity]) + ]; + } + + return $results; + } + + /** + * Build confirmation criteria SQL based on user preferences + */ + private function buildConfirmationCriteria($user_default_confirmation) + { + $criteria = array(); + + if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'Q') !== false) { + $criteria[] = "COL_QSL_RCVD='Y'"; + } + if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'L') !== false) { + $criteria[] = "COL_LOTW_QSL_RCVD='Y'"; + } + if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'E') !== false) { + $criteria[] = "COL_EQSL_QSL_RCVD='Y'"; + } + if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'Z') !== false) { + $criteria[] = "COL_QRZCOM_QSO_DOWNLOAD_STATUS='Y'"; + } + + return empty($criteria) ? '1=0' : '(' . implode(' OR ', $criteria) . ')'; + } + + /** + * Batch query to check which entities have been worked + */ + private function batchWorkedQuery($entities, $logbooks_locations_array) + { + // Use a single query with GROUP BY to check all entities at once + $this->db->select('COL_COUNTRY') + ->distinct() + ->from($this->config->item('table_name')) + ->where('COL_PROP_MODE !=', 'SAT') + ->where_in('station_id', $logbooks_locations_array) + ->where_in('COL_COUNTRY', array_map('urlencode', $entities)); + + $query = $this->db->get(); + $results = array(); + + foreach ($query->result() as $row) { + $results[urldecode($row->COL_COUNTRY)] = true; + } + + return $results; + } + + /** + * Batch query to check which entities have been confirmed + */ + private function batchConfirmedQuery($entities, $logbooks_locations_array, $confirmationCriteria) + { + if ($confirmationCriteria === '1=0') { + return array(); + } + + $this->db->select('COL_COUNTRY') + ->distinct() + ->from($this->config->item('table_name')) + ->where('COL_PROP_MODE !=', 'SAT') + ->where($confirmationCriteria) + ->where_in('station_id', $logbooks_locations_array) + ->where_in('COL_COUNTRY', array_map('urlencode', $entities)); + + $query = $this->db->get(); + $results = array(); + + foreach ($query->result() as $row) { + $results[urldecode($row->COL_COUNTRY)] = true; + } + + return $results; + } public function GetThisWeek() { - $json = file_get_contents($this->optionslib->get_option('dxped_url')); - - // Step 2: Convert the JSON data to an array. + $json = file_get_contents($this->optionslib->get_option('dxped_url')); $data = json_decode($json, true); - // Step 3: Create a new array to hold the records for this week. - $thisWeekRecords = []; + if (empty($data)) { + return array(); + } - // Get the start and end of this week. + $thisWeekRecords = []; $startOfWeek = (new DateTime())->setISODate((new DateTime())->format('o'), (new DateTime())->format('W'), 1); $endOfWeek = (clone $startOfWeek)->modify('+6 days'); - // Step 4: Iterate over the array. - foreach ($data as $record) { + // Get Date format + if ($this->session->userdata('user_date_format')) { + $custom_date_format = $this->session->userdata('user_date_format'); + } else { + $custom_date_format = $this->config->item('qso_date_format'); + } - // Convert "0" and "1" to DateTime objects. + // First pass: filter records for this week + $weekRecords = array(); + foreach ($data as $record) { $startDate = new DateTime($record['0']); $endDate = new DateTime($record['1']); - // Step 5: Check if the start date or end date is within this week. - if (($startDate >= $startOfWeek && $startDate <= $endOfWeek) || ($endDate >= $startOfWeek && $endDate <= $endOfWeek)) { - $endDate = new DateTime($record['1']); - $now = new DateTime(); - $interval = $now->diff($endDate); - $daysLeft = $interval->days; - - // If daysLeft is 0, set it to "Last day" - if ($daysLeft == 0) { - $daysLeft = "Last day"; - } else { - $daysLeft = $daysLeft . " days left"; - } - - // Add daysLeft to record - $record['daysLeft'] = $daysLeft; - // Get Date format - if ($this->session->userdata('user_date_format')) { - // If Logged in and session exists - $custom_date_format = $this->session->userdata('user_date_format'); - } else { - // Get Default date format from /config/cloudlog.php - $custom_date_format = $this->config->item('qso_date_format'); - } - - // Create a new array with the required fields and add it to the main array - $oldStartDate = DateTime::createFromFormat('Y-m-d', $record['0']); - - $StartDate = $oldStartDate->format($custom_date_format); - $record['startDate'] = $StartDate; - - $oldEndDate = DateTime::createFromFormat('Y-m-d', $record['1']); - $EndDate = $oldEndDate->format($custom_date_format); - $record['endDate'] = $EndDate; - - $record['confirmed'] = true; // or false, depending on your logic - - $CI = &get_instance(); - $CI->load->model('logbook_model'); - $dxccInfo = $CI->logbook_model->dxcc_lookup($record['callsign'], $startDate->format('Y-m-d')); - - // Call DXCC Worked function to check if the DXCC has been worked before - if (isset($dxccInfo['entity'])) { - $dxccWorkedData = $this->dxccWorked($dxccInfo['entity']); - $record = array_merge($record, $dxccWorkedData); - } else { - // Handle the case where 'entity' is not set in $dxccInfo - $itemsToAdd = array( - 'workedBefore' => false, - 'confirmed' => false, - ); - $record = array_merge($record, $itemsToAdd); - } - - $thisWeekRecords[] = $record; + if (($startDate >= $startOfWeek && $startDate <= $endOfWeek) || + ($endDate >= $startOfWeek && $endDate <= $endOfWeek)) { + $weekRecords[] = $record; } } - return $thisWeekRecords; + if (empty($weekRecords)) { + return array(); + } + + // Batch process DXCC lookups + $callsigns = array_column($weekRecords, 'callsign'); + $dates = array_column($weekRecords, '0'); + $dxccEntities = $this->batchDxccLookup($callsigns, $dates); + + // Get worked/confirmed status for all entities in batch + $uniqueEntities = array_unique(array_filter($dxccEntities)); + $dxccStatus = $this->batchDxccWorkedStatus($uniqueEntities); + + // Process results + foreach ($weekRecords as $index => $record) { + $endDate = new DateTime($record['1']); + $now = new DateTime(); + $interval = $now->diff($endDate); + $daysLeft = $interval->days; + + $daysLeft = ($daysLeft == 0) ? "Last day" : $daysLeft . " days left"; + $record['daysLeft'] = $daysLeft; + + $oldStartDate = DateTime::createFromFormat('Y-m-d', $record['0']); + $record['startDate'] = $oldStartDate->format($custom_date_format); + + $oldEndDate = DateTime::createFromFormat('Y-m-d', $record['1']); + $record['endDate'] = $oldEndDate->format($custom_date_format); + + // Get DXCC status for this callsign + $entity = $dxccEntities[$index] ?? null; + $worked = $entity && isset($dxccStatus[$entity]) ? $dxccStatus[$entity] : ['workedBefore' => false, 'confirmed' => false]; + + $record['workedBefore'] = $worked['workedBefore']; + $record['confirmed'] = $worked['confirmed']; + + $thisWeekRecords[] = $record; + } + + return $thisWeekRecords; } function dxccWorked($country)