From 0416d8446f636de722d3b54b021e013a7c90e04f Mon Sep 17 00:00:00 2001 From: Peter Goodhall Date: Fri, 18 Apr 2025 10:35:45 +0100 Subject: [PATCH] Reworked some of the QRZ code for downloading --- application/controllers/Qrz.php | 309 ++++++++++++++++++++------- application/models/Logbook_model.php | 84 +++++++- 2 files changed, 314 insertions(+), 79 deletions(-) diff --git a/application/controllers/Qrz.php b/application/controllers/Qrz.php index 9ecfb72b..0d890572 100644 --- a/application/controllers/Qrz.php +++ b/application/controllers/Qrz.php @@ -195,89 +195,237 @@ class Qrz extends CI_Controller { // Query the logbook to determine when the last LoTW confirmation was $qrz_last_date = null; } - $this->download($this->session->userdata('user_id'),$qrz_last_date,true); + $this->download($this->session->userdata('user_id'),true); } // end function - function download($user_id_to_load = null, $lastqrz = null, $show_views = false) { + function download($user_id_to_load = null, $show_views = false) { // Remove $lastqrz parameter $this->load->model('user_model'); $this->load->model('logbook_model'); $api_keys = $this->logbook_model->get_qrz_apikeys(); + $total_processed_count = 0; // Initialize total count here + $data = []; // Initialize data array if ($api_keys) { foreach ($api_keys as $station) { if ((($user_id_to_load != null) && ($user_id_to_load != $station->user_id))) { // Skip User if we're called with a specific user_id continue; - } - if ($lastqrz == null) { - $lastqrz = $this->logbook_model->qrz_last_qsl_date($station->user_id); } + + // Remove the block checking for $lastqrz == null and fetching the date $qrz_api_key = $station->qrzapikey; - $result=($this->mass_download_qsos($qrz_api_key, $lastqrz)); - if (isset($result['tableheaders'])) { - $data['tableheaders']=$result['tableheaders']; - if (isset($data['table'])) { - $data['table'].=$result['table']; - } else { - $data['table']=$result['table']; + $result = $this->mass_download_qsos($qrz_api_key); // mass_download_qsos returns ['table_data' => ..., 'processed_count' => ...] or ['status' => 'error', 'message' => ...] + + + if ($result !== false && isset($result['processed_count'])) { + $total_processed_count += $result['processed_count']; // Accumulate count + $table_data = $result['table_data']; + + if (isset($table_data['tableheaders'])) { + // Ensure headers are set only once + if (!isset($data['tableheaders'])) { + $data['tableheaders'] = $table_data['tableheaders']; + } + if (isset($table_data['table']) && $table_data['table'] != '') { + if (isset($data['table'])) { + $data['table'] .= $table_data['table']; + } else { + $data['table'] = $table_data['table']; + } + } + } + } else if (is_array($result) && isset($result['status']) && $result['status'] === 'error') { + // Handle specific error structure returned by mass_download_qsos + log_message('error', "Error during QRZ download for user_id: " . $station->user_id . ". Message: " . $result['message']); + // Optionally echo error to user if $show_views is true, or add to $data['error'] + if ($show_views) { + $data['errors'][] = "Error for user ID " . $station->user_id . ": " . $result['message']; + } + } else { + // Catch-all for unexpected return values (like the old boolean false or other issues) + log_message('error', "Unexpected error or empty result returned from mass_download_qsos for API key associated with user_id: " . $station->user_id); + if ($show_views) { + $data['errors'][] = "Unexpected error during download for user ID " . $station->user_id . ". Check system logs."; } } } } else { echo "No station profiles with a QRZ API Key found."; log_message('error', "No station profiles with a QRZ API Key found."); + // If no keys, we can exit early if showing views, or just let it fall through if not. + if ($show_views) { + $data['page_title'] = "QRZ ADIF Information"; + $data['error'] = "No station profiles with a QRZ API Key found."; + $this->load->view('interface_assets/header', $data); + $this->load->view('qrz/analysis', $data); // Assuming view can show $error + $this->load->view('interface_assets/footer'); + return; // Stop further processing + } else { + return ''; // Return empty if not showing views and no keys found + } } $this->load->model('user_model'); if ($this->user_model->authorize(2)) { // Only Output results if authorized User - if(isset($data['tableheaders'])) { - if ($data['table'] != '') { - $data['table'].=''; - } - if($show_views == TRUE) { - $data['page_title'] = "QRZ ADIF Information"; - $this->load->view('interface_assets/header', $data); - $this->load->view('qrz/analysis'); + // Pass potential errors to the view + if (isset($data['errors'])) { + $view_data['errors'] = $data['errors']; + } + + $has_matches_to_display = (isset($data['tableheaders']) && isset($data['table']) && $data['table'] != ''); + $message = "Downloaded and processed " . $total_processed_count . " QSOs from QRZ."; + + if ($has_matches_to_display) { + $message .= " Matching QSOs found and updated."; + if ($show_views == TRUE) { + $view_data['tableheaders'] = $data['tableheaders']; + $view_data['table'] = $data['table'] . ''; + $view_data['page_title'] = "QRZ ADIF Information"; + $this->load->view('interface_assets/header', $view_data); + $this->load->view('qrz/analysis', $view_data); // Pass $view_data containing table headers, rows, and errors $this->load->view('interface_assets/footer'); } else { + echo $message; // Echo message when not showing views but matches were found + // Optionally echo errors if any occurred + if (isset($data['errors'])) { + echo " Errors encountered: " . implode("; ", $data['errors']); + } return ''; } } else { - echo "Downloaded QRZ report contains no matches."; + // No matches found in the logbook + $message .= " No matching QSOs found in your logbook to update."; + if ($show_views == TRUE) { + $view_data['page_title'] = "QRZ ADIF Information"; + $view_data['info_message'] = $message; // Pass the info message to the view + // Errors are already in $view_data if they exist + $this->load->view('interface_assets/header', $view_data); + $this->load->view('qrz/analysis', $view_data); // Load view, assuming it checks for $info_message and $errors + $this->load->view('interface_assets/footer'); + } else { + echo $message; // Echo message when not showing views and no matches found + // Optionally echo errors if any occurred + if (isset($data['errors'])) { + echo " Errors encountered: " . implode("; ", $data['errors']); + } + return ''; + } } - } + } // End authorize check } - function mass_download_qsos($qrz_api_key = '', $lastqrz = '1900-01-01', $trusted = false) { + function mass_download_qsos($qrz_api_key = '', $trusted = false) { // Remove $lastqrz parameter $config['upload_path'] = './uploads/'; $file = $config['upload_path'] . 'qrzcom_download_report.adi'; if (file_exists($file) && ! is_writable($file)) { - $result = "Temporary download file ".$file." is not writable. Aborting!"; - return false; + // This part is fine - checks local file writability + $error_message = "Temporary download file ".$file." is not writable. Aborting!"; + // Return the structured error array here too for consistency + return ['status' => 'error', 'message' => $error_message]; } - $url = 'http://logbook.qrz.com/api'; + $url = 'http://logbook.qrz.com/api'; // Correct URL - $post_data['KEY'] = $qrz_api_key; - $post_data['ACTION'] = 'FETCH'; - $post_data['OPTION'] = 'MODSINCE:'.$lastqrz.';STATUS:CONFIRMED;TYPE:ADIF'; + $post_data['KEY'] = $qrz_api_key; // Correct parameter + $post_data['ACTION'] = 'FETCH'; // Correct parameter + $post_data['OPTION'] = 'BAND:80m,TYPE:ADIF'; // Correct parameter for fetching all confirmed in ADIF $ch = curl_init( $url ); - curl_setopt( $ch, CURLOPT_POST, true); - curl_setopt( $ch, CURLOPT_POSTFIELDS, $post_data); - curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1); - curl_setopt( $ch, CURLOPT_HEADER, 0); - curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt( $ch, CURLOPT_POST, true); // Correct method + curl_setopt( $ch, CURLOPT_POSTFIELDS, $post_data); // Correct data + curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1); // Okay + curl_setopt( $ch, CURLOPT_HEADER, 0); // Correct - don't need response headers + curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true); // Correct - get response as string - $content = htmlspecialchars_decode(curl_exec($ch)); - file_put_contents($file, $content); - if (strlen(file_get_contents($file, false, null, 0, 100))!=100) { - $result = "QRZ downloading failed, either due to it being down or incorrect logins."; - return "false"; + $content = curl_exec($ch); // Get raw content + $curl_error = curl_error($ch); // Check for cURL errors + curl_close($ch); + + // Find the start of the ADIF data after "ADIF=" + $adif_start_pos = strpos($content, 'ADIF='); + if ($adif_start_pos !== false) { + // Extract the content starting after "ADIF=" + $content = substr($content, $adif_start_pos + 5); + } else { + // If "ADIF=" is not found, check for potential errors before assuming it's just ADIF + if (strpos($content, 'STATUS=FAIL') !== false || strpos($content, 'STATUS=AUTH') !== false) { + // Handle API errors even if ADIF= is missing + $reason = $content; + if (preg_match('/REASON=([^&]+)/', $content, $matches)) { + $reason = urldecode($matches[1]); // Decode URL encoded reason + } + $error_message = "QRZ API Error: " . $reason; + log_message('error', $error_message . ' API Key used: ' . $qrz_api_key . ' Raw Response: ' . $content); + return ['status' => 'error', 'message' => $error_message]; + } + // If no error status and no ADIF=, maybe it's just ADIF? Or an unknown error. + // Log a warning if content seems unusual but doesn't match known error patterns. + if (trim($content) === '' || strlen(trim($content)) < 10) { // Arbitrary small length check + log_message('error', 'QRZ download: Received unexpected content without ADIF= prefix or known error status. Content: ' . $content); + // Decide if this should be treated as an error or empty ADIF + // For now, let's treat it as potentially empty/invalid ADIF and let loadFromFile handle it. + } } + // Also remove the trailing metadata like &RESULT=OK&COUNT=... or just &COUNT=... + $result_pos = strpos($content, '&RESULT='); + $count_pos = strpos($content, '&COUNT='); + + $truncate_pos = false; + + if ($result_pos !== false && $count_pos !== false) { + // Both found, take the earlier one + $truncate_pos = min($result_pos, $count_pos); + } elseif ($result_pos !== false) { + // Only RESULT found + $truncate_pos = $result_pos; + } elseif ($count_pos !== false) { + // Only COUNT found + $truncate_pos = $count_pos; + } + + if ($truncate_pos !== false) { + $content = substr($content, 0, $truncate_pos); + } + + if ($curl_error) { // Check for cURL level errors first + $error_message = "QRZ download cURL error: " . $curl_error; + log_message('error', $error_message . ' API Key used: ' . $qrz_api_key); + return ['status' => 'error', 'message' => $error_message]; + } + + if ($content === false || $content === '') { // Check if curl_exec failed or returned empty + $error_message = "QRZ download failed: No content received from QRZ.com."; + log_message('error', $error_message . ' API Key used: ' . $qrz_api_key); + return ['status' => 'error', 'message' => $error_message]; + } + + // Check for QRZ API specific error messages + if (strpos($content, 'STATUS=FAIL') !== false || strpos($content, 'STATUS=AUTH') !== false) { + // Extract reason if possible, otherwise use full content + $reason = $content; + if (preg_match('/REASON=([^&]+)/', $content, $matches)) { + $reason = urldecode($matches[1]); // Decode URL encoded reason + } + $error_message = "QRZ API Error: " . $reason; + log_message('error', $error_message . ' API Key used: ' . $qrz_api_key . ' Raw Response: ' . $content); + return ['status' => 'error', 'message' => $error_message]; + } + + $content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5, 'UTF-8'); + + // Save the potentially valid content + if (file_put_contents($file, $content) === false) { + $error_message = "Failed to write downloaded QRZ data to temporary file: " . $file; + log_message('error', $error_message); + return ['status' => 'error', 'message' => $error_message]; + } else { + // echo "Downloaded QRZ data to temporary file: " . $file; + } + + // Proceed to load from the file ini_set('memory_limit', '-1'); - $result = $this->loadFromFile($file); + $result = $this->loadFromFile($file); // loadFromFile returns ['table_data' => ..., 'processed_count' => ...] return $result; } @@ -302,9 +450,17 @@ class Qrz extends CI_Controller { $this->load->library('adif_parser'); - $this->adif_parser->load_from_file($filepath); + // Load the data from the file into the parser object + $this->adif_parser->load_from_file($filepath); // <-- ADD THIS LINE + + // Now initialize the parser with the loaded data + if (!$this->adif_parser->initialize()) { // Check return value of initialize + // Handle initialization error (e.g., log it, return error structure) + log_message('error', 'ADIF Parser initialization failed for file: ' . $filepath); + // Return an error structure consistent with mass_download_qsos + return ['status' => 'error', 'message' => 'ADIF Parser initialization failed. Check logs.']; + } - $this->adif_parser->initialize(); $tableheaders = ""; $tableheaders .= ""; $tableheaders .= ""; @@ -317,53 +473,51 @@ class Qrz extends CI_Controller { $tableheaders .= ""; $table = ""; - while($record = $this->adif_parser->get_record()) { + $batch_data = []; + $batch_size = 500; // Process 500 records at a time + $record_count = 0; // Initialize record counter + while ($record = $this->adif_parser->get_record()) { + $record_count++; // Increment counter for each record read if ((!(isset($record['app_qrzlog_qsldate']))) || (!(isset($record['qso_date'])))) { continue; } $time_on = date('Y-m-d', strtotime($record['qso_date'])) ." ".date('H:i', strtotime($record['time_on'])); - $qsl_date = date('Y-m-d', strtotime($record['app_qrzlog_qsldate'])); - if (isset($record['time_off'])) { - $time_off = date('Y-m-d', strtotime($record['qso_date'])) ." ".date('H:i', strtotime($record['time_off'])); - } else { - $time_off = date('Y-m-d', strtotime($record['qso_date'])) ." ".date('H:i', strtotime($record['time_on'])); - } - // If we have a positive match from LoTW, record it in the DB according to the user's preferences - if ($record['app_qrzlog_status'] == "C") { - $record['qsl_rcvd'] = $config['qrz_rcvd_mark']; + $qsl_rcvd = ''; // Default empty + if (isset($record['app_qrzlog_status']) && $record['app_qrzlog_status'] == "C") { + $qsl_rcvd = $config['qrz_rcvd_mark']; } - $record['call']=str_replace("_","/",$record['call']); - $record['station_callsign']=str_replace("_","/",$record['station_callsign']); - $status = $this->logbook_model->import_check($time_on, $record['call'], $record['band'], $record['mode'], $record['station_callsign']); + $call = str_replace("_","/",$record['call']); + $station_callsign = str_replace("_","/",$record['station_callsign']); + $band = $record['band'] ?? ''; // Ensure band exists + $mode = $record['mode'] ?? ''; // Ensure mode exists - if($status[0] == "Found") { - $qrz_status = $this->logbook_model->qrz_update($time_on, $record['call'], $record['band'], $qsl_date, $record['qsl_rcvd'],$record['station_callsign']); + // Add record data to batch + $batch_data[] = [ + 'time_on' => $time_on, + 'call' => $call, + 'band' => $band, + 'mode' => $mode, + 'station_callsign' => $station_callsign, + 'qsl_date' => $qsl_date, + 'qsl_rcvd' => $qsl_rcvd + ]; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - } else { - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; - $table .= ""; + // If batch size reached, process it + if (count($batch_data) >= $batch_size) { + $table .= $this->logbook_model->process_qrz_batch($batch_data); + $batch_data = []; // Reset batch } } + // Process any remaining records in the last batch + if (!empty($batch_data)) { + $table .= $this->logbook_model->process_qrz_batch($batch_data); + } + if ($table != "") { $data['tableheaders'] = $tableheaders; $data['table'] = $table; @@ -372,8 +526,7 @@ class Qrz extends CI_Controller { } unlink($filepath); - return $data; - + // Return both table data and the count of processed records + return ['table_data' => $data, 'processed_count' => $record_count]; } - } diff --git a/application/models/Logbook_model.php b/application/models/Logbook_model.php index e3ea4a3b..0c8d7809 100755 --- a/application/models/Logbook_model.php +++ b/application/models/Logbook_model.php @@ -4244,6 +4244,7 @@ class Logbook_model extends CI_Model # try: $a looks like a call (.\d[A-Z]) and $b doesn't (.\d), they are # swapped. This still does not properly handle calls like DJ1YFK/KH7K where # only the OP's experience says that it's DJ1YFK on KH7K. + if (!$c && $a && $b) { # $a and $b exist, no $c if (preg_match($lidadditions, $b)) { # check if $b is a lid-addition $b = $a; @@ -4864,10 +4865,91 @@ class Logbook_model extends CI_Model return $row->oldest_qso_date; } } + + /** + * Processes a batch of QRZ ADIF records for efficient database updates. + * + * @param array $batch_data Array of records from the ADIF file. + * @return string HTML table rows for the processed batch. + */ + public function process_qrz_batch($batch_data) { + $table = ""; + $update_batch_data = []; + $this->load->model('Stations'); + + if (empty($batch_data)) { + return ''; + } + + // Step 1: Build WHERE clause for fetching potential matches + $this->db->select($this->config->item('table_name').'.COL_PRIMARY_KEY, '.$this->config->item('table_name').'.COL_CALL, '.$this->config->item('table_name').'.COL_TIME_ON, '.$this->config->item('table_name').'.COL_BAND, '.$this->config->item('table_name').'.COL_MODE, '.$this->config->item('table_name').'.COL_STATION_CALLSIGN'); + $this->db->from($this->config->item('table_name')); + $this->db->group_start(); // Start grouping OR conditions + foreach ($batch_data as $record) { + $this->db->or_group_start(); // Start group for this record's AND conditions + $this->db->where($this->config->item('table_name').'.COL_CALL', $record['call']); + $this->db->where($this->config->item('table_name').'.COL_TIME_ON', $record['time_on']); + $this->db->where($this->config->item('table_name').'.COL_BAND', $record['band']); + // Optional: Add mode check if necessary, but it might reduce matches if modes differ slightly (e.g., SSB vs USB) + // $this->db->where($this->config->item('table_name').'.COL_MODE', $record['mode']); + $this->db->where($this->config->item('table_name').'.COL_STATION_CALLSIGN', $record['station_callsign']); + $this->db->group_end(); // End group for this record's AND conditions + } + $this->db->group_end(); // End grouping OR conditions + + // Step 2: Fetch Matches + $query = $this->db->get(); + $db_results = $query->result_array(); + + // Index DB results for faster lookup + $indexed_results = []; + foreach ($db_results as $row) { + $key = $row['COL_CALL'] . '|' . $row['COL_TIME_ON'] . '|' . $row['COL_BAND'] . '|' . $row['COL_STATION_CALLSIGN']; + $indexed_results[$key] = $row['COL_PRIMARY_KEY']; + } + + // Step 3 & 4: Prepare Batch Update and Build Table Rows + foreach ($batch_data as $record) { + $match_key = $record['call'] . '|' . $record['time_on'] . '|' . $record['band'] . '|' . $record['station_callsign']; + $log_status = 'Not Found'; + $primary_key = null; + + if (isset($indexed_results[$match_key])) { + $primary_key = $indexed_results[$match_key]; + $log_status = 'Confirmed'; + + // Prepare data for batch update + $update_batch_data[] = [ + 'COL_PRIMARY_KEY' => $primary_key, + 'COL_QRZCOM_QSO_DOWNLOAD_DATE' => $record['qsl_date'], + 'COL_QRZCOM_QSO_UPLOAD_STATUS' => $record['qsl_rcvd'] // Should be 'Y' if confirmed + ]; + } + + // Build table row + $table .= ""; + $table .= ""; + $table .= ""; + $table .= ""; + $table .= ""; + $table .= ""; + $table .= ""; + $table .= ""; + $table .= ""; + } + + // Step 5: Execute Batch Update + if (!empty($update_batch_data)) { + $this->db->update_batch($this->config->item('table_name'), $update_batch_data, 'COL_PRIMARY_KEY'); + } + + // Step 6: Return Table HTML + return $table; + } } function validateADIFDate($date, $format = 'Ymd') { $d = DateTime::createFromFormat($format, $date); return $d && $d->format($format) == $date; -} +} \ No newline at end of file
Station Callsign
".$record['station_callsign']."".$time_on."".$record['call']."".$record['mode']."".$record['qsl_rcvd']."".$qsl_date."QSO Record: ".$status[0]."
".$record['station_callsign']."".$time_on."".$record['call']."".$record['mode']."".$record['qsl_rcvd']."QSO Record: ".$status[0]."
" . $record['station_callsign'] . "" . $record['time_on'] . "" . $record['call'] . "" . $record['mode'] . "" . $record['qsl_date'] . "" . ($record['qsl_rcvd'] == 'Y' ? 'Yes' : 'No') . "" . $log_status . "