[LoTW] Added table with users. Query to find unconfirmed QSOs and L marking in advanced logbook

application/config/migration.php

@@ -21,7 +21,7 @@ $config['migration_enabled'] = TRUE;
 | be upgraded / downgraded to.
 |
 */
-$config['migration_version'] = 122;
+$config['migration_version'] = 123;
 
 /*
 |--------------------------------------------------------------------------

application/controllers/Logbook.php

@@ -814,6 +814,42 @@ class Logbook extends CI_Controller {
 
 	}
 
+	function search_lotw_unconfirmed($station_id) {
+		$station_id = $this->security->xss_clean($station_id);
+
+		$this->load->model('user_model');
+
+		if(!$this->user_model->authorize($this->config->item('auth_mode'))) { return; }
+		
+		$CI =& get_instance();
+		$CI->load->model('logbooks_model');
+		$logbooks_locations_array = $CI->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook'));
+		
+		if (!$logbooks_locations_array) {
+			return null;
+		}
+
+		$location_list = "'".implode("','",$logbooks_locations_array)."'";
+		
+		$sql = 'select COL_CALL, COL_MODE, COL_SUBMODE, station_callsign, COL_SAT_NAME, COL_BAND, COL_TIME_ON, lotw_users.lastupload from ' . $this->config->item('table_name') . 
+		' join station_profile on ' . $this->config->item('table_name') . '.station_id = station_profile.station_id 
+		join lotw_users on ' . $this->config->item('table_name') . '.col_call = lotw_users.callsign 
+		where ' . $this->config->item('table_name') .'.station_id in ('. $location_list . ')'; 
+		
+		if ($station_id != 'All') {
+			$sql .= ' and station_profile.station_id = ' . $station_id;
+		}
+		
+		$sql .= " and COL_LOTW_QSL_RCVD <> 'Y' and " . $this->config->item('table_name') . ".COL_TIME_ON < lotw_users.lastupload";
+
+		$query = $this->db->query($sql);
+
+		$data['qsos'] = $query;
+
+		$this->load->view('search/lotw_unconfirmed_result.php', $data);
+
+	}
+
 	function search_incorrect_cq_zones($station_id) {
 		$station_id = $this->security->xss_clean($station_id);
 

application/controllers/Search.php

@@ -80,6 +80,18 @@ class Search extends CI_Controller {
 		$this->load->view('interface_assets/footer');
 	}
 
+	// Searches for unconfirmed Lotw QSOs where QSO partner has uploaded to LoTW after the QSO date
+	public function lotw_unconfirmed() {
+		$this->load->model('stations');
+
+		$data['station_profile'] = $this->stations->all_of_user();
+		$data['page_title'] = "QSOs unconfirmed on LoTW, but the callsign has uploaded to LoTW after QSO date";
+
+		$this->load->view('interface_assets/header', $data);
+		$this->load->view('search/lotw_unconfirmed');
+		$this->load->view('interface_assets/footer');
+	}
+
     function json_result() {
           if(isset($_POST['search'])) {
 			  $result = $this->fetchQueryResult($_POST['search'], false);

application/controllers/Update.php

@@ -287,9 +287,6 @@ class Update extends CI_Controller {
     }
 
     public function download_lotw_users() {
-
-
-
         $contents = file_get_contents('https://lotw.arrl.org/lotw-user-activity.csv', true);
 
         if($contents === FALSE) { 
@@ -306,6 +303,47 @@ class Update extends CI_Controller {
 
     }
 
+    public function lotw_users() {
+        $mtime = microtime(); 
+        $mtime = explode(" ",$mtime); 
+        $mtime = $mtime[1] + $mtime[0]; 
+        $starttime = $mtime; 
+
+        $file = 'https://lotw.arrl.org/lotw-user-activity.csv';
+
+        $handle = fopen($file, "r");
+        if ($handle === FALSE) {
+            echo "Something went wrong with fetching the LoTW uses file";
+            return;
+        }
+        $this->db->empty_table("lotw_users"); 
+        $i = 0;
+        $data = fgetcsv($handle,1000,",");
+        do {
+            if ($data[0]) {
+                $lotwdata[$i]['callsign'] = $data[0];
+                $lotwdata[$i]['lastupload'] = $data[1] . ' ' . $data[2];
+                if (($i % 2000) == 0) {
+                    $this->db->insert_batch('lotw_users', $lotwdata); 
+                    unset($lotwdata);
+                    // echo 'Record ' . $i . '<br />';
+                }
+                $i++;
+            }
+        } while ($data = fgetcsv($handle,1000,","));
+        fclose($handle);
+
+        $this->db->insert_batch('lotw_users', $lotwdata); 
+
+        $mtime = microtime(); 
+        $mtime = explode(" ",$mtime); 
+        $mtime = $mtime[1] + $mtime[0]; 
+        $endtime = $mtime; 
+        $totaltime = ($endtime - $starttime); 
+        echo "This page was created in ".$totaltime." seconds <br />"; 
+        echo "Records inserted: " . $i . " <br/>";
+    }
+
     public function lotw_check() {
         $f = fopen('./updates/lotw_users.csv', "r");
         $result = false;

application/migrations/123_add_lotw_users.php

@@ -0,0 +1,41 @@
+<?php
+defined('BASEPATH') or exit('No direct script access allowed');
+
+class Migration_add_lotw_users extends CI_Migration
+{
+	public function up()
+	{
+		if (!$this->db->table_exists('lotw_users')) {
+            $this->dbforge->add_field(array(
+                'id' => array(
+                    'type' => 'INT',
+                    'constraint' => 20,
+                    'unsigned' => TRUE,
+                    'auto_increment' => TRUE,
+                    'unique' => TRUE
+                ),
+                'callsign' => array(
+                    'type' => 'VARCHAR',
+                    'constraint' => 32,
+                    'unsigned' => TRUE,
+                ),
+                'lastupload' => array(
+                    'type' => 'timestamp',
+                    'unsigned' => TRUE,
+                )
+            ));
+
+            $this->dbforge->add_key('id', TRUE);
+
+            $this->dbforge->create_table('lotw_users');
+
+            $this->db->query("ALTER TABLE lotw_users ADD INDEX `callsign` (`callsign`)");
+        }
+
+	}
+
+	public function down()
+	{
+        $this->dbforge->drop_table('lotw_users');
+	}
+}

application/models/Logbookadvanced_model.php

@@ -100,6 +100,7 @@ class Logbookadvanced_model extends CI_Model {
 			FROM " . $this->config->item('table_name') . " qsos
 			INNER JOIN station_profile ON qsos.station_id=station_profile.station_id
 			LEFT OUTER JOIN dxcc_entities ON qsos.col_dxcc=dxcc_entities.adif
+			LEFT OUTER JOIN lotw_users ON qsos.col_call=lotw_users.callsign
 			WHERE station_profile.user_id =  ?
 			$where
 			ORDER BY qsos.COL_TIME_ON desc, qsos.COL_PRIMARY_KEY desc

application/views/interface_assets/footer.php

@@ -816,6 +816,29 @@ function findduplicates(){
     });
 }
 
+function findlotwunconfirmed(){
+    event.preventDefault();
+    $('#partial_view').load(base_url+"index.php/logbook/search_lotw_unconfirmed/"+$("#station_id").val(), function() {
+        $('.qsolist').DataTable({
+            "pageLength": 25,
+            responsive: false,
+            ordering: false,
+            "scrollY":        "500px",
+            "scrollCollapse": true,
+            "paging":         false,
+            "scrollX": true,
+            dom: 'Bfrtip',
+            buttons: [
+                'csv'
+            ]
+        });
+        // change color of csv-button if dark mode is chosen
+        if (isDarkModeTheme()) {
+            $(".buttons-csv").css("color", "white");
+        }
+    });
+}
+
 function findincorrectcqzones() {
     event.preventDefault();
     $('#partial_view').load(base_url+"index.php/logbook/search_incorrect_cq_zones/"+$("#station_id").val(), function() {

application/views/search/cqzones.php

@@ -17,9 +17,11 @@
 		  <li class="nav-item">
 	        <a class="nav-link" href="<?php echo site_url('search/duplicates'); ?>">Duplicate QSOs</a>
 	      </li>
-		  </li>
 		  <li class="nav-item">
 	        <a class="nav-link active" href="<?php echo site_url('search/incorrect_cq_zones'); ?>">Incorrect CQ Zones</a>
+	      </li>
+		  <li class="nav-item">
+	        <a class="nav-link" href="<?php echo site_url('search/lotw_unconfirmed'); ?>">QSOS unconfirmed on LoTW</a>
 	      </li>
 	    </ul>
 	  </div>

application/views/search/duplicates.php

@@ -17,9 +17,11 @@
 		  <li class="nav-item">
 	        <a class="nav-link active" href="<?php echo site_url('search/duplicates'); ?>">Duplicate QSOs</a>
 	      </li>
-		  </li>
 		  <li class="nav-item">
 	        <a class="nav-link" href="<?php echo site_url('search/incorrect_cq_zones'); ?>">Incorrect CQ Zones</a>
+	      </li>
+		  <li class="nav-item">
+	        <a class="nav-link" href="<?php echo site_url('search/lotw_unconfirmed'); ?>">QSOS unconfirmed on LoTW</a>
 	      </li>
 	    </ul>
 	  </div>

application/views/search/filter.php

@@ -25,9 +25,11 @@
 		  <li class="nav-item">
 	        <a class="nav-link" href="<?php echo site_url('search/duplicates'); ?>">Duplicate QSOs</a>
 	      </li>
-		  </li>
 		  <li class="nav-item">
 	        <a class="nav-link" href="<?php echo site_url('search/incorrect_cq_zones'); ?>">Incorrect CQ Zones</a>
+	      </li>
+		  <li class="nav-item">
+	        <a class="nav-link" href="<?php echo site_url('search/lotw_unconfirmed'); ?>">QSOS unconfirmed on LoTW</a>
 	      </li>
 	    </ul>
 	  </div>

application/views/search/lotw_unconfirmed.php

@@ -0,0 +1,50 @@
+<div class="container search">
+
+	<h1>
+		Search
+		<small class="text-muted">Ready to find a QSO?</small>
+	</h1>
+
+	<div class="card text-center">
+	  <div class="card-header">
+	    <ul class="nav nav-tabs card-header-tabs">
+	      <li class="nav-item">
+	        <a class="nav-link" href="<?php echo site_url('search'); ?>">Search</a>
+	      </li>
+	      <li class="nav-item">
+	        <a class="nav-link" href="<?php echo site_url('search/filter'); ?>">Advanced Search</a>
+	      </li>
+		  <li class="nav-item">
+	        <a class="nav-link" href="<?php echo site_url('search/duplicates'); ?>">Duplicate QSOs</a>
+	      </li>
+		  <li class="nav-item">
+	        <a class="nav-link " href="<?php echo site_url('search/incorrect_cq_zones'); ?>">Incorrect CQ Zones</a>
+	      </li>
+		  <li class="nav-item">
+	        <a class="nav-link active" href="<?php echo site_url('search/lotw_unconfirmed'); ?>">QSOS unconfirmed on LoTW</a>
+	      </li>
+	    </ul>
+	  </div>
+	  <div class="card-body">
+        The search displays QSOs which are unconfirmed on LoTW, but the callsign worked has uploaded to LoTW after your QSO date.<br/><br />
+	  	<form method="post" action="" id="search_box" name="test">
+		  <div class="form-group row">
+		    <label for="callsign" class="col-sm-2 col-form-label">Station location:</label>
+		    <select id="station_id" name="station_profile" class="custom-select col-sm-3 mb-3 mr-sm-3">
+					<option value="All">All</option>
+                    <?php foreach ($station_profile->result() as $station) { ?>
+                    <option value="<?php echo $station->station_id; ?>">Callsign: <?php echo $station->station_callsign; ?> (<?php echo $station->station_profile_name; ?>)</option>
+                    <?php } ?>
+                    </select>
+		    <div class="col-sm-2">
+		    	<button onclick="findlotwunconfirmed();" class="btn btn-outline-success my-2 my-sm-0" type="submit"><i class="fas fa-search"></i> Search</button>
+		    </div>
+		  </div>
+		</form>
+
+		<div id="partial_view"></div>
+
+	  </div>
+	</div>
+
+</div>

application/views/search/lotw_unconfirmed_result.php

@@ -0,0 +1,43 @@
+<?php
+if ($qsos->result() != NULL) {
+	echo '<table style="width:100%" class="qsolist table-sm table-bordered table-hover table-striped table-condensed">
+	<thead>
+	<tr>
+		<th style=\'text-align: center\'>'.lang('gen_hamradio_callsign').'</th>
+		<th style=\'text-align: center\'>Date / time</th>
+		<th style=\'text-align: center\'>' . lang('gen_hamradio_mode') . '</th>
+		<th style=\'text-align: center\'>' . lang('gen_hamradio_band') . '</th>
+		<th style=\'text-align: center\'>Last LoTW upload</th>
+		<th style=\'text-align: center\'>' . lang('gen_hamradio_station') . '</th>
+	</tr>
+	</thead><tbody>';
+
+	// Get Date format
+	if($this->session->userdata('user_date_format')) {
+		// If Logged in and session exists
+		$custom_date_format = $this->session->userdata('user_date_format');
+	} else {
+		// Get Default date format from /config/cloudlog.php
+		$custom_date_format = $this->config->item('qso_date_format');
+	}
+
+	$i = 0;
+	foreach ($qsos->result() as $qso) {
+		echo '<tr>';
+		echo '<td style=\'text-align: center\'><form id="searchcall_'.$i.'" method="POST" action="'.site_url('search'). '"><input type="hidden" value="'. strtoupper($qso->COL_CALL) .'" name="callsign"><a href="javascript:$(\'#searchcall_'.$i++.'\').submit()">' . $qso->COL_CALL . '</a></form></td>';
+		echo '<td style=\'text-align: center\'>' . $qso->COL_TIME_ON . '</td>';
+		echo '<td style=\'text-align: center\'>'; echo $qso->COL_SUBMODE==null?$qso->COL_MODE:$qso->COL_SUBMODE; echo '</td>';
+		echo '<td style=\'text-align: center\'>'; if($qso->COL_SAT_NAME != null) { echo $qso->COL_SAT_NAME; } else { echo strtolower($qso->COL_BAND); }; echo '</td>';
+		echo '<td style=\'text-align: center\'>' . $qso->lastupload . '</td>';
+		echo '<td style=\'text-align: center\'><span class="badge badge-light">' . $qso->station_callsign . '</span></td>';
+		echo '</tr>';
+	}
+
+	echo '</tbody></table>';
+	?>
+
+	<?php
+} else {
+	echo '<div class="alert alert-success"><a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>No duplicate QSO\'s were found.</div>';
+}
+?>

application/views/search/main.php

@@ -19,6 +19,9 @@
 	      </li>
 		  <li class="nav-item">
 	        <a class="nav-link" href="<?php echo site_url('search/incorrect_cq_zones'); ?>">Incorrect CQ Zones</a>
+	      </li>
+		  <li class="nav-item">
+	        <a class="nav-link" href="<?php echo site_url('search/lotw_unconfirmed'); ?>">QSOS unconfirmed on LoTW</a>
 	      </li>
 	    </ul>
 	  </div>

assets/js/sections/logbookadvanced.js

@@ -26,7 +26,7 @@ function updateRow(qso) {
 	let c = 1;
 	cells.eq(c++).text(qso.qsoDateTime);
 	cells.eq(c++).text(qso.de);
-	cells.eq(c++).html('<a id="edit_qso" href="javascript:displayQso('+qso.qsoID+')">'+qso.dx+'</a>');
+	cells.eq(c++).html('<a id="edit_qso" href="javascript:displayQso('+qso.qsoID+')">'+qso.dx+'</a>' + (qso.callsign == '' ? '' : ' <small id="lotw_info" class="badge badge-success" data-toggle="tooltip" data-original-title="LoTW User. Last upload was ' + qso.lastupload + '">L</small>'));
 	cells.eq(c++).text(qso.mode);
 	cells.eq(c++).text(qso.rstS);
 	cells.eq(c++).text(qso.rstR);
@@ -81,7 +81,7 @@ function loadQSOTable(rows) {
 		data.push('<div class="form-check"><input class="form-check-input" type="checkbox" /></div>');
 		data.push(qso.qsoDateTime);
 		data.push(qso.de);
-		data.push('<a id="edit_qso" href="javascript:displayQso('+qso.qsoID+')">'+qso.dx+'</a>');
+		data.push('<a id="edit_qso" href="javascript:displayQso('+qso.qsoID+')">'+qso.dx+'</a>' + (qso.callsign == '' ? '' : ' <small id="lotw_info" class="badge badge-success" data-toggle="tooltip" data-original-title="LoTW User. Last upload was ' + qso.lastupload + ' ">L</small>'));
 		data.push(qso.mode);
 		data.push(qso.rstS);
 		data.push(qso.rstR);

src/QSLManager/QSO.php

@@ -59,6 +59,9 @@ class QSO
 	private string $qsl;
 	private string $lotw;
 	private string $eqsl;
+	/** Lotw callsign info **/
+	private string $callsign;
+	private string $lastupload;
 
 	/**
 	 * @param array $data Does no validation, it's assumed to be a row from the database in array format
@@ -191,6 +194,8 @@ class QSO
 		} else {
 			$this->end = null;
 		}
+		$this->callsign = ($data['callsign'] === null) ? '' :$data['callsign'];
+		$this->lastupload = ($data['lastupload'] === null) ? '' :$data['lastupload'];
 	}
 	
 	/**
@@ -769,6 +774,8 @@ class QSO
 			'cqzone' => $this->getCqzone(),
 			'iota' => $this->getIOTA(),
 			'end' => $this->end === null ? null : $this->end->format("Y-m-d"),
+			'callsign' => $this->callsign,
+			'lastupload' => $this->lastupload,
 		];
 	}