ADIF Import functions tuned to increase processing speed

application/config/migration.php

@@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE;
 |
 */
 
-$config['migration_version'] = 162;
+$config['migration_version'] = 163;
 
 /*
 |--------------------------------------------------------------------------

application/controllers/Adif.php

@@ -158,9 +158,10 @@ class adif extends CI_Controller {
 	public function import() {
 		$this->load->model('stations');
 		$data['station_profile'] = $this->stations->all_of_user();
+		log_message("debug","Started ADIF Import");
 
-        $active_station_id = $this->stations->find_active();
+        	$active_station_id = $this->stations->find_active();
-        $station_profile = $this->stations->profile($active_station_id);
+        	$station_profile = $this->stations->profile($active_station_id);
 
 		$data['active_station_info'] = $station_profile->row();
 
@@ -183,7 +184,6 @@ class adif extends CI_Controller {
 		} else {
 			if ($this->stations->check_station_is_accessible($this->input->post('station_profile'))) {
 				$data = array('upload_data' => $this->upload->data());
-
 				ini_set('memory_limit', '-1');
 				set_time_limit(0);
 
@@ -192,21 +192,21 @@ class adif extends CI_Controller {
 				$this->load->library('adif_parser');
 
 				$this->adif_parser->load_from_file('./uploads/'.$data['upload_data']['file_name']);
+				unlink('./uploads/'.$data['upload_data']['file_name']);
+				$data['upload_data']='';	// free memory
 
 				$this->adif_parser->initialize();
 				$custom_errors = "";
+				$alladif=[];
 				while($record = $this->adif_parser->get_record())
 				{
 					if(count($record) == 0) {
 						break;
 					};
-
+					array_push($alladif,$record);
-					$one_error = $this->logbook_model->import($record, $this->input->post('station_profile'), $this->input->post('skipDuplicate'), $this->input->post('markClublog'),$this->input->post('markLotw'), $this->input->post('dxccAdif'), $this->input->post('markQrz'), $this->input->post('markHrd'), true, $this->input->post('operatorName'), false, $this->input->post('skipStationCheck'));
-					if ($one_error != '') {
-						$custom_errors.=$one_error."<br/>";
-					}
 				};
-				unlink('./uploads/'.$data['upload_data']['file_name']);
+				$record='';	// free memory
+				$custom_errors = $this->logbook_model->import_bulk($alladif, $this->input->post('station_profile'), $this->input->post('skipDuplicate'), $this->input->post('markClublog'),$this->input->post('markLotw'), $this->input->post('dxccAdif'), $this->input->post('markQrz'), $this->input->post('markHrd'), true, $this->input->post('operatorName'), false, $this->input->post('skipStationCheck'));
 			} else {
 				$custom_errors='Station Profile not valid for User';
 			}
@@ -214,7 +214,7 @@ class adif extends CI_Controller {
 			$data['adif_errors'] = $custom_errors;
 			$data['skip_dupes'] = $this->input->post('skipDuplicate');
 
-
+			log_message("debug","Finished ADIF Import");
 			$data['page_title'] = "ADIF Imported";
 			$this->load->view('interface_assets/header', $data);
 			$this->load->view('adif/import_success');

application/migrations/163_dxcc_index.php

@@ -0,0 +1,16 @@
+<?php
+
+defined('BASEPATH') OR exit('No direct script access allowed');
+
+class Migration_dxcc_index extends CI_Migration {
+
+	public function up() {
+		$this->db->query("ALTER TABLE `dxcc_prefixes` ADD INDEX `idx_dxcc_prefixes_logic` (`call`, `start`, `end`)");
+		$this->db->query("ALTER TABLE `dxcc_exceptions` ADD INDEX `idx_dxcc_exceptions_logic` (`call`, `start`, `end`)");
+	}
+
+	public function down(){
+		$this->db->query("ALTER TABLE dxcc_prefixes DROP INDEX idx_dxcc_prefixes_logic");
+		$this->db->query("ALTER TABLE dxcc_exceptions DROP INDEX idx_dxcc_exceptions_logic");
+	}
+}

application/models/Logbook_model.php

@@ -3077,6 +3077,16 @@ function lotw_last_qsl_date($user_id) {
 	  return '1900-01-01 00:00:00.000';
   }
 
+  function import_bulk($records, $station_id = "0", $skipDuplicate = false, $markClublog = false, $markLotw = false, $dxccAdif = false, $markQrz = false, $markHrd = false,$skipexport = false, $operatorName = false, $apicall = false, $skipStationCheck = false) {
+	  $custom_errors='';
+	  foreach ($records as $record) {
+		  $one_error = $this->logbook_model->import($record, $station_id, $skipDuplicate, $markClublog, $markLotw,$dxccAdif, $markQrz, $markHrd, $skipexport, $operatorName, $apicall, $skipStationCheck);
+		  if ($one_error != '') {
+			  $custom_errors.=$one_error."<br/>";
+		  }
+	  }
+	  return $custom_errors;
+  }
     /*
      * $skipDuplicate - used in ADIF import to skip duplicate checking when importing QSOs
      * $markLoTW - used in ADIF import to mark QSOs as exported to LoTW when importing QSOs
@@ -3087,13 +3097,12 @@ function lotw_last_qsl_date($user_id) {
      */
 	function import($record, $station_id = "0", $skipDuplicate = false, $markClublog = false, $markLotw = false, $dxccAdif = false, $markQrz = false, $markHrd = false,$skipexport = false, $operatorName = false, $apicall = false, $skipStationCheck = false) {
         // be sure that station belongs to user
-        $CI =& get_instance();
+        $this->load->model('stations');
-        $CI->load->model('stations');
+        if (!$this->stations->check_station_is_accessible($station_id) && $apicall == false ) {
-        if (!$CI->stations->check_station_is_accessible($station_id) && $apicall == false ) {
             return 'Station not accessible<br>';
         }
 
-	$station_profile=$CI->stations->profile_clean($station_id);
+	$station_profile=$this->stations->profile_clean($station_id);
 	$station_profile_call=$station_profile->station_callsign;
 
 	if (($station_id !=0 ) && (!(isset($record['station_callsign'])))) {
@@ -3105,8 +3114,7 @@ function lotw_last_qsl_date($user_id) {
               "<br>See the <a target=\"_blank\" href=\"https://github.com/magicbug/Cloudlog/wiki/ADIF-file-can't-be-imported\">Cloudlog Wiki</a> for hints about errors in ADIF files.";
         }
 
-        $CI =& get_instance();
+        $this->load->library('frequency');
-        $CI->load->library('frequency');
         $my_error = "";
 
         // Join date+time
@@ -3188,7 +3196,7 @@ function lotw_last_qsl_date($user_id) {
         } else {
             if (isset($record['freq'])){
               if($freq != "0") {
-                $band = $CI->frequency->GetBand($freq);
+                $band = $this->frequency->GetBand($freq);
               }
             }
         }
@@ -3198,7 +3206,7 @@ function lotw_last_qsl_date($user_id) {
         } else {
                 if (isset($record['freq_rx'])){
                   if($freq != "0") {
-                    $band_rx = $CI->frequency->GetBand($freqRX);
+                    $band_rx = $this->frequency->GetBand($freqRX);
                   }
                 } else {
                   $band_rx = "";
@@ -3424,9 +3432,8 @@ function lotw_last_qsl_date($user_id) {
 
         // Get active station_id from station profile if one hasn't been provided
         if($station_id == "" || $station_id == "0") {
-          $CI =& get_instance();
+          $this->load->model('stations');
-          $CI->load->model('stations');
+          $station_id = $this->stations->find_active();
-          $station_id = $CI->stations->find_active();
         }
 
         // Check if QSO is already in the database
@@ -3682,10 +3689,9 @@ function lotw_last_qsl_date($user_id) {
     }
 
     function update_dok($record, $ignoreAmbiguous, $onlyConfirmed, $overwriteDok) {
-        $CI =& get_instance();
+        $this->load->model('logbooks_model');
-        $CI->load->model('logbooks_model');
         $custom_date_format = $this->session->userdata('user_date_format');
-        $logbooks_locations_array = $CI->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook'));
+        $logbooks_locations_array = $this->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook'));
 
         if(isset($record['call'])) {
                 $call = strtoupper($record['call']);
@@ -3702,7 +3708,7 @@ function lotw_last_qsl_date($user_id) {
         } else {
             if (isset($record['freq'])){
               if($record['freq'] != "0") {
-                $band = $CI->frequency->GetBand($record['freq']);
+                $band = $this->frequency->GetBand($record['freq']);
               }
             }
         }