goodspeed/Cloudlog
1
0
派生 0
代码 工单 合并请求 项目 版本发布 软件包 百科 动态 Actions

Reworked some of the QRZ code for downloading

浏览代码
这个提交包含在：
Peter Goodhall 2025-04-18 10:35:45 +01:00
父节点 b1a68d6c61
当前提交 0416d8446f
共有 2 个文件被更改，包括 314 次插入 和 79 次删除
显示统计信息 下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

301
application/controllers/Qrz.php
查看文件

@ -195,89 +195,237 @@ class Qrz extends CI_Controller {
// Query the logbook to determine when the last LoTW confirmation was // Query the logbook to determine when the last LoTW confirmation was
$qrz_last_date = null; $qrz_last_date = null;
} }
$this->download($this->session->userdata('user_id'),$qrz_last_date,true); $this->download($this->session->userdata('user_id'),true);
} // end function } // end function
function download($user_id_to_load = null, $lastqrz = null, $show_views = false) { function download($user_id_to_load = null, $show_views = false) { // Remove $lastqrz parameter
$this->load->model('user_model'); $this->load->model('user_model');
$this->load->model('logbook_model'); $this->load->model('logbook_model');
$api_keys = $this->logbook_model->get_qrz_apikeys(); $api_keys = $this->logbook_model->get_qrz_apikeys();
$total_processed_count = 0; // Initialize total count here
$data = []; // Initialize data array
if ($api_keys) { if ($api_keys) {
foreach ($api_keys as $station) { foreach ($api_keys as $station) {
if ((($user_id_to_load != null) && ($user_id_to_load != $station->user_id))) { // Skip User if we're called with a specific user_id if ((($user_id_to_load != null) && ($user_id_to_load != $station->user_id))) { // Skip User if we're called with a specific user_id
continue; continue;
} }
if ($lastqrz == null) {
$lastqrz = $this->logbook_model->qrz_last_qsl_date($station->user_id); // Remove the block checking for $lastqrz == null and fetching the date
}
$qrz_api_key = $station->qrzapikey; $qrz_api_key = $station->qrzapikey;
$result=($this->mass_download_qsos($qrz_api_key, $lastqrz)); $result = $this->mass_download_qsos($qrz_api_key); // mass_download_qsos returns ['table_data' => ..., 'processed_count' => ...] or ['status' => 'error', 'message' => ...]
if (isset($result['tableheaders'])) {
$data['tableheaders']=$result['tableheaders'];
if ($result !== false && isset($result['processed_count'])) {
$total_processed_count += $result['processed_count']; // Accumulate count
$table_data = $result['table_data'];
if (isset($table_data['tableheaders'])) {
// Ensure headers are set only once
if (!isset($data['tableheaders'])) {
$data['tableheaders'] = $table_data['tableheaders'];
}
if (isset($table_data['table']) && $table_data['table'] != '') {
if (isset($data['table'])) { if (isset($data['table'])) {
$data['table'].=$result['table']; $data['table'] .= $table_data['table'];
} else { } else {
$data['table']=$result['table']; $data['table'] = $table_data['table'];
}
}
}
} else if (is_array($result) && isset($result['status']) && $result['status'] === 'error') {
// Handle specific error structure returned by mass_download_qsos
log_message('error', "Error during QRZ download for user_id: " . $station->user_id . ". Message: " . $result['message']);
// Optionally echo error to user if $show_views is true, or add to $data['error']
if ($show_views) {
$data['errors'][] = "Error for user ID " . $station->user_id . ": " . $result['message'];
}
} else {
// Catch-all for unexpected return values (like the old boolean false or other issues)
log_message('error', "Unexpected error or empty result returned from mass_download_qsos for API key associated with user_id: " . $station->user_id);
if ($show_views) {
$data['errors'][] = "Unexpected error during download for user ID " . $station->user_id . ". Check system logs.";
} }
} }
} }
} else { } else {
echo "No station profiles with a QRZ API Key found."; echo "No station profiles with a QRZ API Key found.";
log_message('error', "No station profiles with a QRZ API Key found."); log_message('error', "No station profiles with a QRZ API Key found.");
// If no keys, we can exit early if showing views, or just let it fall through if not.
if ($show_views) {
$data['page_title'] = "QRZ ADIF Information";
$data['error'] = "No station profiles with a QRZ API Key found.";
$this->load->view('interface_assets/header', $data);
$this->load->view('qrz/analysis', $data); // Assuming view can show $error
$this->load->view('interface_assets/footer');
return; // Stop further processing
} else {
return ''; // Return empty if not showing views and no keys found
}
} }
$this->load->model('user_model'); $this->load->model('user_model');
if ($this->user_model->authorize(2)) { // Only Output results if authorized User if ($this->user_model->authorize(2)) { // Only Output results if authorized User
if(isset($data['tableheaders'])) { // Pass potential errors to the view
if ($data['table'] != '') { if (isset($data['errors'])) {
$data['table'].='</table>'; $view_data['errors'] = $data['errors'];
} }
if($show_views == TRUE) {
$data['page_title'] = "QRZ ADIF Information"; $has_matches_to_display = (isset($data['tableheaders']) && isset($data['table']) && $data['table'] != '');
$this->load->view('interface_assets/header', $data); $message = "Downloaded and processed " . $total_processed_count . " QSOs from QRZ.";
$this->load->view('qrz/analysis');
if ($has_matches_to_display) {
$message .= " Matching QSOs found and updated.";
if ($show_views == TRUE) {
$view_data['tableheaders'] = $data['tableheaders'];
$view_data['table'] = $data['table'] . '</table>';
$view_data['page_title'] = "QRZ ADIF Information";
$this->load->view('interface_assets/header', $view_data);
$this->load->view('qrz/analysis', $view_data); // Pass $view_data containing table headers, rows, and errors
$this->load->view('interface_assets/footer'); $this->load->view('interface_assets/footer');
} else { } else {
echo $message; // Echo message when not showing views but matches were found
// Optionally echo errors if any occurred
if (isset($data['errors'])) {
echo " Errors encountered: " . implode("; ", $data['errors']);
}
return ''; return '';
} }
} else { } else {
echo "Downloaded QRZ report contains no matches."; // No matches found in the logbook
$message .= " No matching QSOs found in your logbook to update.";
if ($show_views == TRUE) {
$view_data['page_title'] = "QRZ ADIF Information";
$view_data['info_message'] = $message; // Pass the info message to the view
// Errors are already in $view_data if they exist
$this->load->view('interface_assets/header', $view_data);
$this->load->view('qrz/analysis', $view_data); // Load view, assuming it checks for $info_message and $errors
$this->load->view('interface_assets/footer');
} else {
echo $message; // Echo message when not showing views and no matches found
// Optionally echo errors if any occurred
if (isset($data['errors'])) {
echo " Errors encountered: " . implode("; ", $data['errors']);
}
return '';
} }
} }
} // End authorize check
} }
function mass_download_qsos($qrz_api_key = '', $lastqrz = '1900-01-01', $trusted = false) { function mass_download_qsos($qrz_api_key = '', $trusted = false) { // Remove $lastqrz parameter
$config['upload_path'] = './uploads/'; $config['upload_path'] = './uploads/';
$file = $config['upload_path'] . 'qrzcom_download_report.adi'; $file = $config['upload_path'] . 'qrzcom_download_report.adi';
if (file_exists($file) && ! is_writable($file)) { if (file_exists($file) && ! is_writable($file)) {
$result = "Temporary download file ".$file." is not writable. Aborting!"; // This part is fine - checks local file writability
return false; $error_message = "Temporary download file ".$file." is not writable. Aborting!";
// Return the structured error array here too for consistency
return ['status' => 'error', 'message' => $error_message];
} }
$url = 'http://logbook.qrz.com/api'; $url = 'http://logbook.qrz.com/api'; // Correct URL
$post_data['KEY'] = $qrz_api_key; $post_data['KEY'] = $qrz_api_key; // Correct parameter
$post_data['ACTION'] = 'FETCH'; $post_data['ACTION'] = 'FETCH'; // Correct parameter
$post_data['OPTION'] = 'MODSINCE:'.$lastqrz.';STATUS:CONFIRMED;TYPE:ADIF'; $post_data['OPTION'] = 'BAND:80m,TYPE:ADIF'; // Correct parameter for fetching all confirmed in ADIF
$ch = curl_init( $url ); $ch = curl_init( $url );
curl_setopt( $ch, CURLOPT_POST, true); curl_setopt( $ch, CURLOPT_POST, true); // Correct method
curl_setopt( $ch, CURLOPT_POSTFIELDS, $post_data); curl_setopt( $ch, CURLOPT_POSTFIELDS, $post_data); // Correct data
curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1); // Okay
curl_setopt( $ch, CURLOPT_HEADER, 0); curl_setopt( $ch, CURLOPT_HEADER, 0); // Correct - don't need response headers
curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true); curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true); // Correct - get response as string
$content = htmlspecialchars_decode(curl_exec($ch)); $content = curl_exec($ch); // Get raw content
file_put_contents($file, $content); $curl_error = curl_error($ch); // Check for cURL errors
if (strlen(file_get_contents($file, false, null, 0, 100))!=100) { curl_close($ch);
$result = "QRZ downloading failed, either due to it being down or incorrect logins.";
return "false"; // Find the start of the ADIF data after "ADIF="
$adif_start_pos = strpos($content, 'ADIF=');
if ($adif_start_pos !== false) {
// Extract the content starting after "ADIF="
$content = substr($content, $adif_start_pos + 5);
} else {
// If "ADIF=" is not found, check for potential errors before assuming it's just ADIF
if (strpos($content, 'STATUS=FAIL') !== false || strpos($content, 'STATUS=AUTH') !== false) {
// Handle API errors even if ADIF= is missing
$reason = $content;
if (preg_match('/REASON=([^&]+)/', $content, $matches)) {
$reason = urldecode($matches[1]); // Decode URL encoded reason
}
$error_message = "QRZ API Error: " . $reason;
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key . ' Raw Response: ' . $content);
return ['status' => 'error', 'message' => $error_message];
}
// If no error status and no ADIF=, maybe it's just ADIF? Or an unknown error.
// Log a warning if content seems unusual but doesn't match known error patterns.
if (trim($content) === '' || strlen(trim($content)) < 10) { // Arbitrary small length check
log_message('error', 'QRZ download: Received unexpected content without ADIF= prefix or known error status. Content: ' . $content);
// Decide if this should be treated as an error or empty ADIF
// For now, let's treat it as potentially empty/invalid ADIF and let loadFromFile handle it.
}
} }
// Also remove the trailing metadata like &RESULT=OK&COUNT=... or just &COUNT=...
$result_pos = strpos($content, '&RESULT=');
$count_pos = strpos($content, '&COUNT=');
$truncate_pos = false;
if ($result_pos !== false && $count_pos !== false) {
// Both found, take the earlier one
$truncate_pos = min($result_pos, $count_pos);
} elseif ($result_pos !== false) {
// Only RESULT found
$truncate_pos = $result_pos;
} elseif ($count_pos !== false) {
// Only COUNT found
$truncate_pos = $count_pos;
}
if ($truncate_pos !== false) {
$content = substr($content, 0, $truncate_pos);
}
if ($curl_error) { // Check for cURL level errors first
$error_message = "QRZ download cURL error: " . $curl_error;
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key);
return ['status' => 'error', 'message' => $error_message];
}
if ($content === false || $content === '') { // Check if curl_exec failed or returned empty
$error_message = "QRZ download failed: No content received from QRZ.com.";
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key);
return ['status' => 'error', 'message' => $error_message];
}
// Check for QRZ API specific error messages
if (strpos($content, 'STATUS=FAIL') !== false || strpos($content, 'STATUS=AUTH') !== false) {
// Extract reason if possible, otherwise use full content
$reason = $content;
if (preg_match('/REASON=([^&]+)/', $content, $matches)) {
$reason = urldecode($matches[1]); // Decode URL encoded reason
}
$error_message = "QRZ API Error: " . $reason;
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key . ' Raw Response: ' . $content);
return ['status' => 'error', 'message' => $error_message];
}
$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5, 'UTF-8');
// Save the potentially valid content
if (file_put_contents($file, $content) === false) {
$error_message = "Failed to write downloaded QRZ data to temporary file: " . $file;
log_message('error', $error_message);
return ['status' => 'error', 'message' => $error_message];
} else {
// echo "Downloaded QRZ data to temporary file: " . $file;
}
// Proceed to load from the file
ini_set('memory_limit', '-1'); ini_set('memory_limit', '-1');
$result = $this->loadFromFile($file); $result = $this->loadFromFile($file); // loadFromFile returns ['table_data' => ..., 'processed_count' => ...]
return $result; return $result;
} }
@ -302,9 +450,17 @@ class Qrz extends CI_Controller {
$this->load->library('adif_parser'); $this->load->library('adif_parser');
$this->adif_parser->load_from_file($filepath); // Load the data from the file into the parser object
$this->adif_parser->load_from_file($filepath); // <-- ADD THIS LINE
// Now initialize the parser with the loaded data
if (!$this->adif_parser->initialize()) { // Check return value of initialize
// Handle initialization error (e.g., log it, return error structure)
log_message('error', 'ADIF Parser initialization failed for file: ' . $filepath);
// Return an error structure consistent with mass_download_qsos
return ['status' => 'error', 'message' => 'ADIF Parser initialization failed. Check logs.'];
}
$this->adif_parser->initialize();
$tableheaders = "<table width=\"100%\">"; $tableheaders = "<table width=\"100%\">";
$tableheaders .= "<tr class=\"titles\">"; $tableheaders .= "<tr class=\"titles\">";
$tableheaders .= "<td>Station Callsign</td>"; $tableheaders .= "<td>Station Callsign</td>";
@ -317,53 +473,51 @@ class Qrz extends CI_Controller {
$tableheaders .= "</tr>"; $tableheaders .= "</tr>";
$table = ""; $table = "";
while($record = $this->adif_parser->get_record()) { $batch_data = [];
$batch_size = 500; // Process 500 records at a time
$record_count = 0; // Initialize record counter
while ($record = $this->adif_parser->get_record()) {
$record_count++; // Increment counter for each record read
if ((!(isset($record['app_qrzlog_qsldate']))) || (!(isset($record['qso_date'])))) { if ((!(isset($record['app_qrzlog_qsldate']))) || (!(isset($record['qso_date'])))) {
continue; continue;
} }
$time_on = date('Y-m-d', strtotime($record['qso_date'])) ." ".date('H:i', strtotime($record['time_on'])); $time_on = date('Y-m-d', strtotime($record['qso_date'])) ." ".date('H:i', strtotime($record['time_on']));
$qsl_date = date('Y-m-d', strtotime($record['app_qrzlog_qsldate'])); $qsl_date = date('Y-m-d', strtotime($record['app_qrzlog_qsldate']));
if (isset($record['time_off'])) {
$time_off = date('Y-m-d', strtotime($record['qso_date'])) ." ".date('H:i', strtotime($record['time_off']));
} else {
$time_off = date('Y-m-d', strtotime($record['qso_date'])) ." ".date('H:i', strtotime($record['time_on']));
}
// If we have a positive match from LoTW, record it in the DB according to the user's preferences // If we have a positive match from LoTW, record it in the DB according to the user's preferences
if ($record['app_qrzlog_status'] == "C") { $qsl_rcvd = ''; // Default empty
$record['qsl_rcvd'] = $config['qrz_rcvd_mark']; if (isset($record['app_qrzlog_status']) && $record['app_qrzlog_status'] == "C") {
$qsl_rcvd = $config['qrz_rcvd_mark'];
} }
$record['call']=str_replace("_","/",$record['call']); $call = str_replace("_","/",$record['call']);
$record['station_callsign']=str_replace("_","/",$record['station_callsign']); $station_callsign = str_replace("_","/",$record['station_callsign']);
$status = $this->logbook_model->import_check($time_on, $record['call'], $record['band'], $record['mode'], $record['station_callsign']); $band = $record['band'] ?? ''; // Ensure band exists
$mode = $record['mode'] ?? ''; // Ensure mode exists
if($status[0] == "Found") { // Add record data to batch
$qrz_status = $this->logbook_model->qrz_update($time_on, $record['call'], $record['band'], $qsl_date, $record['qsl_rcvd'],$record['station_callsign']); $batch_data[] = [
'time_on' => $time_on,
'call' => $call,
'band' => $band,
'mode' => $mode,
'station_callsign' => $station_callsign,
'qsl_date' => $qsl_date,
'qsl_rcvd' => $qsl_rcvd
];
$table .= "<tr>"; // If batch size reached, process it
$table .= "<td>".$record['station_callsign']."</td>"; if (count($batch_data) >= $batch_size) {
$table .= "<td>".$time_on."</td>"; $table .= $this->logbook_model->process_qrz_batch($batch_data);
$table .= "<td>".$record['call']."</td>"; $batch_data = []; // Reset batch
$table .= "<td>".$record['mode']."</td>";
$table .= "<td>".$record['qsl_rcvd']."</td>";
$table .= "<td>".$qsl_date."</td>";
$table .= "<td>QSO Record: ".$status[0]."</td>";
$table .= "</tr>";
} else {
$table .= "<tr>";
$table .= "<td>".$record['station_callsign']."</td>";
$table .= "<td>".$time_on."</td>";
$table .= "<td>".$record['call']."</td>";
$table .= "<td>".$record['mode']."</td>";
$table .= "<td>".$record['qsl_rcvd']."</td>";
$table .= "<td>QSO Record: ".$status[0]."</td>";
$table .= "</tr>";
} }
} }
// Process any remaining records in the last batch
if (!empty($batch_data)) {
$table .= $this->logbook_model->process_qrz_batch($batch_data);
}
if ($table != "") { if ($table != "") {
$data['tableheaders'] = $tableheaders; $data['tableheaders'] = $tableheaders;
$data['table'] = $table; $data['table'] = $table;
@ -372,8 +526,7 @@ class Qrz extends CI_Controller {
} }
unlink($filepath); unlink($filepath);
return $data; // Return both table data and the count of processed records
return ['table_data' => $data, 'processed_count' => $record_count];
} }
} }

82
application/models/Logbook_model.php
查看文件

@ -4244,6 +4244,7 @@ class Logbook_model extends CI_Model
# try: $a looks like a call (.\d[A-Z]) and $b doesn't (.\d), they are # try: $a looks like a call (.\d[A-Z]) and $b doesn't (.\d), they are
# swapped. This still does not properly handle calls like DJ1YFK/KH7K where # swapped. This still does not properly handle calls like DJ1YFK/KH7K where
# only the OP's experience says that it's DJ1YFK on KH7K. # only the OP's experience says that it's DJ1YFK on KH7K.
if (!$c && $a && $b) { # $a and $b exist, no $c if (!$c && $a && $b) { # $a and $b exist, no $c
if (preg_match($lidadditions, $b)) { # check if $b is a lid-addition if (preg_match($lidadditions, $b)) { # check if $b is a lid-addition
$b = $a; $b = $a;
@ -4864,6 +4865,87 @@ class Logbook_model extends CI_Model
return $row->oldest_qso_date; return $row->oldest_qso_date;
} }
} }
/**
* Processes a batch of QRZ ADIF records for efficient database updates.
*
* @param array $batch_data Array of records from the ADIF file.
* @return string HTML table rows for the processed batch.
*/
public function process_qrz_batch($batch_data) {
$table = "";
$update_batch_data = [];
$this->load->model('Stations');
if (empty($batch_data)) {
return '';
}
// Step 1: Build WHERE clause for fetching potential matches
$this->db->select($this->config->item('table_name').'.COL_PRIMARY_KEY, '.$this->config->item('table_name').'.COL_CALL, '.$this->config->item('table_name').'.COL_TIME_ON, '.$this->config->item('table_name').'.COL_BAND, '.$this->config->item('table_name').'.COL_MODE, '.$this->config->item('table_name').'.COL_STATION_CALLSIGN');
$this->db->from($this->config->item('table_name'));
$this->db->group_start(); // Start grouping OR conditions
foreach ($batch_data as $record) {
$this->db->or_group_start(); // Start group for this record's AND conditions
$this->db->where($this->config->item('table_name').'.COL_CALL', $record['call']);
$this->db->where($this->config->item('table_name').'.COL_TIME_ON', $record['time_on']);
$this->db->where($this->config->item('table_name').'.COL_BAND', $record['band']);
// Optional: Add mode check if necessary, but it might reduce matches if modes differ slightly (e.g., SSB vs USB)
// $this->db->where($this->config->item('table_name').'.COL_MODE', $record['mode']);
$this->db->where($this->config->item('table_name').'.COL_STATION_CALLSIGN', $record['station_callsign']);
$this->db->group_end(); // End group for this record's AND conditions
}
$this->db->group_end(); // End grouping OR conditions
// Step 2: Fetch Matches
$query = $this->db->get();
$db_results = $query->result_array();
// Index DB results for faster lookup
$indexed_results = [];
foreach ($db_results as $row) {
$key = $row['COL_CALL'] . '|' . $row['COL_TIME_ON'] . '|' . $row['COL_BAND'] . '|' . $row['COL_STATION_CALLSIGN'];
$indexed_results[$key] = $row['COL_PRIMARY_KEY'];
}
// Step 3 & 4: Prepare Batch Update and Build Table Rows
foreach ($batch_data as $record) {
$match_key = $record['call'] . '|' . $record['time_on'] . '|' . $record['band'] . '|' . $record['station_callsign'];
$log_status = '<span class="badge text-bg-danger">Not Found</span>';
$primary_key = null;
if (isset($indexed_results[$match_key])) {
$primary_key = $indexed_results[$match_key];
$log_status = '<span class="badge text-bg-success">Confirmed</span>';
// Prepare data for batch update
$update_batch_data[] = [
'COL_PRIMARY_KEY' => $primary_key,
'COL_QRZCOM_QSO_DOWNLOAD_DATE' => $record['qsl_date'],
'COL_QRZCOM_QSO_UPLOAD_STATUS' => $record['qsl_rcvd'] // Should be 'Y' if confirmed
];
}
// Build table row
$table .= "<tr>";
$table .= "<td>" . $record['station_callsign'] . "</td>";
$table .= "<td>" . $record['time_on'] . "</td>";
$table .= "<td>" . $record['call'] . "</td>";
$table .= "<td>" . $record['mode'] . "</td>";
$table .= "<td>" . $record['qsl_date'] . "</td>";
$table .= "<td>" . ($record['qsl_rcvd'] == 'Y' ? '<span class="badge text-bg-success">Yes</span>' : '<span class="badge text-bg-danger">No</span>') . "</td>";
$table .= "<td>" . $log_status . "</td>";
$table .= "</tr>";
}
// Step 5: Execute Batch Update
if (!empty($update_batch_data)) {
$this->db->update_batch($this->config->item('table_name'), $update_batch_data, 'COL_PRIMARY_KEY');
}
// Step 6: Return Table HTML
return $table;
}
} }
function validateADIFDate($date, $format = 'Ymd') function validateADIFDate($date, $format = 'Ymd')