From 412e74f027caf1d88cbef7c4c2fdd5c1b2bb5ee9 Mon Sep 17 00:00:00 2001 From: int2001 Date: Sun, 17 Dec 2023 09:01:09 +0000 Subject: [PATCH 1/5] 2nd try to speed up importer --- application/controllers/Adif.php | 14 +++++++++----- application/models/Logbook_model.php | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/application/controllers/Adif.php b/application/controllers/Adif.php index 5992bfdb..55f8c8c3 100644 --- a/application/controllers/Adif.php +++ b/application/controllers/Adif.php @@ -195,18 +195,22 @@ class adif extends CI_Controller { $this->adif_parser->initialize(); $custom_errors = ""; + $alladif=[]; while($record = $this->adif_parser->get_record()) { if(count($record) == 0) { break; }; - - $one_error = $this->logbook_model->import($record, $this->input->post('station_profile'), $this->input->post('skipDuplicate'), $this->input->post('markClublog'),$this->input->post('markLotw'), $this->input->post('dxccAdif'), $this->input->post('markQrz'), $this->input->post('markHrd'), true, $this->input->post('operatorName'), false, $this->input->post('skipStationCheck')); - if ($one_error != '') { - $custom_errors.=$one_error."
"; - } + array_push($alladif,$record); + // $one_error = $this->logbook_model->import($record, $this->input->post('station_profile'), $this->input->post('skipDuplicate'), $this->input->post('markClublog'),$this->input->post('markLotw'), $this->input->post('dxccAdif'), $this->input->post('markQrz'), $this->input->post('markHrd'), true, $this->input->post('operatorName'), false, $this->input->post('skipStationCheck')); + // if ($one_error != '') { + // $custom_errors.=$one_error."
"; + // } }; unlink('./uploads/'.$data['upload_data']['file_name']); + $record=''; + $data=''; + $custom_errors = $this->logbook_model->import_bulk($alladif, $this->input->post('station_profile'), $this->input->post('skipDuplicate'), $this->input->post('markClublog'),$this->input->post('markLotw'), $this->input->post('dxccAdif'), $this->input->post('markQrz'), $this->input->post('markHrd'), true, $this->input->post('operatorName'), false, $this->input->post('skipStationCheck')); } else { $custom_errors='Station Profile not valid for User'; } diff --git a/application/models/Logbook_model.php b/application/models/Logbook_model.php index bf659eea..47a76ee4 100755 --- a/application/models/Logbook_model.php +++ b/application/models/Logbook_model.php @@ -3077,6 +3077,16 @@ function lotw_last_qsl_date($user_id) { return '1900-01-01 00:00:00.000'; } + function import_bulk($records, $station_id = "0", $skipDuplicate = false, $markClublog = false, $markLotw = false, $dxccAdif = false, $markQrz = false, $markHrd = false,$skipexport = false, $operatorName = false, $apicall = false, $skipStationCheck = false) { + $custom_errors=''; + foreach ($records as $record) { + $one_error = $this->logbook_model->import($record, $station_id, $skipDuplicate, $markClublog, $markLotw,$dxccadif, $markQrz, $markHrd, $skipexport, $operatorName, $apicall, $skipStationCheck); + if ($one_error != '') { + $custom_errors.=$one_error."
"; + } + } + return $custom_errors; + } /* * $skipDuplicate - used in ADIF import to skip duplicate checking when importing QSOs * $markLoTW - used in ADIF import to mark QSOs as exported to LoTW when importing QSOs From 3c44382f231fe8755be1a55683baa3480255ded4 Mon Sep 17 00:00:00 2001 From: int2001 Date: Sun, 17 Dec 2023 10:24:13 +0000 Subject: [PATCH 2/5] TuneUp Import by creating INDEX on DXCC-Tables and altering logic --- application/config/migration.php | 2 +- application/controllers/Adif.php | 13 ++++--------- application/models/Logbook_model.php | 28 ++++++++++++---------------- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/application/config/migration.php b/application/config/migration.php index ed1cc07a..3441f9fa 100644 --- a/application/config/migration.php +++ b/application/config/migration.php @@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE; | */ -$config['migration_version'] = 162; +$config['migration_version'] = 163; /* |-------------------------------------------------------------------------- diff --git a/application/controllers/Adif.php b/application/controllers/Adif.php index 55f8c8c3..533e8a10 100644 --- a/application/controllers/Adif.php +++ b/application/controllers/Adif.php @@ -183,7 +183,6 @@ class adif extends CI_Controller { } else { if ($this->stations->check_station_is_accessible($this->input->post('station_profile'))) { $data = array('upload_data' => $this->upload->data()); - ini_set('memory_limit', '-1'); set_time_limit(0); @@ -192,6 +191,8 @@ class adif extends CI_Controller { $this->load->library('adif_parser'); $this->adif_parser->load_from_file('./uploads/'.$data['upload_data']['file_name']); + unlink('./uploads/'.$data['upload_data']['file_name']); + $data['upload_data']=''; // free memory $this->adif_parser->initialize(); $custom_errors = ""; @@ -202,14 +203,8 @@ class adif extends CI_Controller { break; }; array_push($alladif,$record); - // $one_error = $this->logbook_model->import($record, $this->input->post('station_profile'), $this->input->post('skipDuplicate'), $this->input->post('markClublog'),$this->input->post('markLotw'), $this->input->post('dxccAdif'), $this->input->post('markQrz'), $this->input->post('markHrd'), true, $this->input->post('operatorName'), false, $this->input->post('skipStationCheck')); - // if ($one_error != '') { - // $custom_errors.=$one_error."
"; - // } }; - unlink('./uploads/'.$data['upload_data']['file_name']); - $record=''; - $data=''; + $record=''; // free memory $custom_errors = $this->logbook_model->import_bulk($alladif, $this->input->post('station_profile'), $this->input->post('skipDuplicate'), $this->input->post('markClublog'),$this->input->post('markLotw'), $this->input->post('dxccAdif'), $this->input->post('markQrz'), $this->input->post('markHrd'), true, $this->input->post('operatorName'), false, $this->input->post('skipStationCheck')); } else { $custom_errors='Station Profile not valid for User'; @@ -218,7 +213,7 @@ class adif extends CI_Controller { $data['adif_errors'] = $custom_errors; $data['skip_dupes'] = $this->input->post('skipDuplicate'); - + log_message("Error","Finished ADIF Import"); $data['page_title'] = "ADIF Imported"; $this->load->view('interface_assets/header', $data); $this->load->view('adif/import_success'); diff --git a/application/models/Logbook_model.php b/application/models/Logbook_model.php index 47a76ee4..26a53d7f 100755 --- a/application/models/Logbook_model.php +++ b/application/models/Logbook_model.php @@ -3080,7 +3080,7 @@ function lotw_last_qsl_date($user_id) { function import_bulk($records, $station_id = "0", $skipDuplicate = false, $markClublog = false, $markLotw = false, $dxccAdif = false, $markQrz = false, $markHrd = false,$skipexport = false, $operatorName = false, $apicall = false, $skipStationCheck = false) { $custom_errors=''; foreach ($records as $record) { - $one_error = $this->logbook_model->import($record, $station_id, $skipDuplicate, $markClublog, $markLotw,$dxccadif, $markQrz, $markHrd, $skipexport, $operatorName, $apicall, $skipStationCheck); + $one_error = $this->logbook_model->import($record, $station_id, $skipDuplicate, $markClublog, $markLotw,$dxccAdif, $markQrz, $markHrd, $skipexport, $operatorName, $apicall, $skipStationCheck); if ($one_error != '') { $custom_errors.=$one_error."
"; } @@ -3097,13 +3097,12 @@ function lotw_last_qsl_date($user_id) { */ function import($record, $station_id = "0", $skipDuplicate = false, $markClublog = false, $markLotw = false, $dxccAdif = false, $markQrz = false, $markHrd = false,$skipexport = false, $operatorName = false, $apicall = false, $skipStationCheck = false) { // be sure that station belongs to user - $CI =& get_instance(); - $CI->load->model('stations'); - if (!$CI->stations->check_station_is_accessible($station_id) && $apicall == false ) { + $this->load->model('stations'); + if (!$this->stations->check_station_is_accessible($station_id) && $apicall == false ) { return 'Station not accessible
'; } - $station_profile=$CI->stations->profile_clean($station_id); + $station_profile=$this->stations->profile_clean($station_id); $station_profile_call=$station_profile->station_callsign; if (($station_id !=0 ) && (!(isset($record['station_callsign'])))) { @@ -3115,8 +3114,7 @@ function lotw_last_qsl_date($user_id) { "
See the Cloudlog Wiki for hints about errors in ADIF files."; } - $CI =& get_instance(); - $CI->load->library('frequency'); + $this->load->library('frequency'); $my_error = ""; // Join date+time @@ -3198,7 +3196,7 @@ function lotw_last_qsl_date($user_id) { } else { if (isset($record['freq'])){ if($freq != "0") { - $band = $CI->frequency->GetBand($freq); + $band = $this->frequency->GetBand($freq); } } } @@ -3208,7 +3206,7 @@ function lotw_last_qsl_date($user_id) { } else { if (isset($record['freq_rx'])){ if($freq != "0") { - $band_rx = $CI->frequency->GetBand($freqRX); + $band_rx = $this->frequency->GetBand($freqRX); } } else { $band_rx = ""; @@ -3434,9 +3432,8 @@ function lotw_last_qsl_date($user_id) { // Get active station_id from station profile if one hasn't been provided if($station_id == "" || $station_id == "0") { - $CI =& get_instance(); - $CI->load->model('stations'); - $station_id = $CI->stations->find_active(); + $this->load->model('stations'); + $station_id = $this->stations->find_active(); } // Check if QSO is already in the database @@ -3692,10 +3689,9 @@ function lotw_last_qsl_date($user_id) { } function update_dok($record, $ignoreAmbiguous, $onlyConfirmed, $overwriteDok) { - $CI =& get_instance(); - $CI->load->model('logbooks_model'); + $this->load->model('logbooks_model'); $custom_date_format = $this->session->userdata('user_date_format'); - $logbooks_locations_array = $CI->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook')); + $logbooks_locations_array = $this->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook')); if(isset($record['call'])) { $call = strtoupper($record['call']); @@ -3712,7 +3708,7 @@ function lotw_last_qsl_date($user_id) { } else { if (isset($record['freq'])){ if($record['freq'] != "0") { - $band = $CI->frequency->GetBand($record['freq']); + $band = $this->frequency->GetBand($record['freq']); } } } From 903d2248ca4926152cd4f8f7812f847293e94ceb Mon Sep 17 00:00:00 2001 From: int2001 Date: Sun, 17 Dec 2023 10:31:02 +0000 Subject: [PATCH 3/5] lowered debugoutput from "Error" to debug --- application/controllers/Adif.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/application/controllers/Adif.php b/application/controllers/Adif.php index 533e8a10..64750fc5 100644 --- a/application/controllers/Adif.php +++ b/application/controllers/Adif.php @@ -158,9 +158,10 @@ class adif extends CI_Controller { public function import() { $this->load->model('stations'); $data['station_profile'] = $this->stations->all_of_user(); + log_message("debug","Started ADIF Import"); - $active_station_id = $this->stations->find_active(); - $station_profile = $this->stations->profile($active_station_id); + $active_station_id = $this->stations->find_active(); + $station_profile = $this->stations->profile($active_station_id); $data['active_station_info'] = $station_profile->row(); @@ -213,7 +214,7 @@ class adif extends CI_Controller { $data['adif_errors'] = $custom_errors; $data['skip_dupes'] = $this->input->post('skipDuplicate'); - log_message("Error","Finished ADIF Import"); + log_message("debug","Finished ADIF Import"); $data['page_title'] = "ADIF Imported"; $this->load->view('interface_assets/header', $data); $this->load->view('adif/import_success'); From c0f08d2be7f302ae72e75cbc2dbfab36510d470e Mon Sep 17 00:00:00 2001 From: int2001 Date: Sun, 17 Dec 2023 10:32:09 +0000 Subject: [PATCH 4/5] Added Mig for IDX --- application/migrations/163_dxcc_index.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 application/migrations/163_dxcc_index.php diff --git a/application/migrations/163_dxcc_index.php b/application/migrations/163_dxcc_index.php new file mode 100644 index 00000000..7b456be7 --- /dev/null +++ b/application/migrations/163_dxcc_index.php @@ -0,0 +1,16 @@ +db->query("ALTER TABLE `dxcc_prefixes` ADD INDEX `idx_dxcc_prefixes_logic` (`call`, `start`, `end`)"); + $this->db->query("ALTER TABLE `dxcc_exceptions` ADD INDEX `idx_dxcc_exceptions_logic` (`call`, `start`, `end`)"); + } + + public function down(){ + $this->db->query("ALTER TABLE dxcc_prefixes DROP INDEX idx_dxcc_prefixes_logic"); + $this->db->query("ALTER TABLE dxcc_exceptions DROP INDEX idx_dxcc_eceptions_logic"); + } +} From d9dc458658f3087bf30c05bdf4aeb0f1d2fc2801 Mon Sep 17 00:00:00 2001 From: int2001 Date: Sun, 17 Dec 2023 10:35:31 +0000 Subject: [PATCH 5/5] Fixed typo in migscript --- application/migrations/163_dxcc_index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/migrations/163_dxcc_index.php b/application/migrations/163_dxcc_index.php index 7b456be7..f847b5ae 100644 --- a/application/migrations/163_dxcc_index.php +++ b/application/migrations/163_dxcc_index.php @@ -11,6 +11,6 @@ class Migration_dxcc_index extends CI_Migration { public function down(){ $this->db->query("ALTER TABLE dxcc_prefixes DROP INDEX idx_dxcc_prefixes_logic"); - $this->db->query("ALTER TABLE dxcc_exceptions DROP INDEX idx_dxcc_eceptions_logic"); + $this->db->query("ALTER TABLE dxcc_exceptions DROP INDEX idx_dxcc_exceptions_logic"); } }