diff --git a/application/config/migration.php b/application/config/migration.php index 343f22cd0..9948fe947 100644 --- a/application/config/migration.php +++ b/application/config/migration.php @@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE; | */ -$config['migration_version'] = 246; +$config['migration_version'] = 247; /* |-------------------------------------------------------------------------- diff --git a/application/controllers/Lotw.php b/application/controllers/Lotw.php index 1a0aaf37c..56cb66215 100644 --- a/application/controllers/Lotw.php +++ b/application/controllers/Lotw.php @@ -526,6 +526,10 @@ class Lotw extends CI_Controller { $tableheaders .= ""; $table = ""; + $batch_updates = array(); // Collect all updates for batch processing + $table_rows = array(); // Collect table rows for later rendering + + // First pass: collect all records for batch processing while($record = $this->adif_parser->get_record()) { @@ -548,75 +552,93 @@ class Lotw extends CI_Controller { $status = $this->logbook_model->import_check($time_on, $record['call'], $record['band'], $record['mode'], $record['station_callsign']); if($status[0] == "Found") { - if (isset($record['state'])) { - $state = $record['state']; - } else { - $state = ""; - } - // Present only if the QSLing station specified a single valid grid square value in its station location uploaded to LoTW. - if (isset($record['gridsquare'])) { - $qsl_gridsquare = $record['gridsquare']; - } else { - $qsl_gridsquare = ""; - } + $state = isset($record['state']) ? $record['state'] : ""; + $qsl_gridsquare = isset($record['gridsquare']) ? $record['gridsquare'] : ""; + $qsl_vucc_grids = isset($record['vucc_grids']) ? $record['vucc_grids'] : ""; + $iota = isset($record['iota']) ? $record['iota'] : ""; + $cnty = isset($record['cnty']) ? $record['cnty'] : ""; + $cqz = isset($record['cqz']) ? $record['cqz'] : ""; + $ituz = isset($record['ituz']) ? $record['ituz'] : ""; - if (isset($record['vucc_grids'])) { - $qsl_vucc_grids = $record['vucc_grids']; - } else { - $qsl_vucc_grids = ""; - } - - if (isset($record['iota'])) { - $iota = $record['iota']; - } else { - $iota = ""; - } - - if (isset($record['cnty'])) { - $cnty = $record['cnty']; - } else { - $cnty = ""; - } - - if (isset($record['cqz'])) { - $cqz = $record['cqz']; - } else { - $cqz = ""; - } - - if (isset($record['ituz'])) { - $ituz = $record['ituz']; - } else { - $ituz = ""; - } - - $lotw_status = $this->logbook_model->lotw_update($time_on, $record['call'], $record['band'], $qsl_date, $record['qsl_rcvd'], $state, $qsl_gridsquare, $qsl_vucc_grids, $iota, $cnty, $cqz, $ituz, $record['station_callsign']); + // Add to batch update array + $batch_updates[] = array( + 'datetime' => $time_on, + 'callsign' => $record['call'], + 'band' => $record['band'], + 'qsl_date' => $qsl_date, + 'qsl_status' => $record['qsl_rcvd'], + 'state' => $state, + 'qsl_gridsquare' => $qsl_gridsquare, + 'qsl_vucc_grids' => $qsl_vucc_grids, + 'iota' => $iota, + 'cnty' => $cnty, + 'cqz' => $cqz, + 'ituz' => $ituz, + 'station_callsign' => $record['station_callsign'] + ); + $table_rows[] = array( + 'station_callsign' => $record['station_callsign'], + 'time_on' => $time_on, + 'call' => $record['call'], + 'mode' => $record['mode'], + 'qsl_rcvd' => $record['qsl_rcvd'], + 'qsl_date' => $qsl_date, + 'state' => $state, + 'gridsquare' => ($qsl_gridsquare != '' ? $qsl_gridsquare : $qsl_vucc_grids), + 'iota' => $iota, + 'status' => $status[0], + 'found' => true + ); + } else { + $table_rows[] = array( + 'station_callsign' => $record['station_callsign'], + 'time_on' => $time_on, + 'call' => $record['call'], + 'mode' => $record['mode'], + 'qsl_rcvd' => $record['qsl_rcvd'], + 'status' => $status[0], + 'found' => false + ); + } + } + + // Batch update all LOTW confirmations in one operation + $lotw_status = "No updates"; + if (!empty($batch_updates)) { + $result = $this->logbook_model->lotw_update_batch($batch_updates); + $lotw_status = "Batch Updated: {$result['updated']} QSOs, {$result['gridsquare_updated']} gridsquares"; + log_message('info', 'LoTW Download: ' . $lotw_status); + } + + // Build table from collected rows + foreach ($table_rows as $row) { + if ($row['found']) { $table .= ""; - $table .= "".$record['station_callsign'].""; - $table .= "".$time_on.""; - $table .= "".$record['call'].""; - $table .= "".$record['mode'].""; - $table .= "".$record['qsl_rcvd'].""; - $table .= "".$qsl_date.""; - $table .= "".$state.""; - $table .= "".($qsl_gridsquare != '' ? $qsl_gridsquare : $qsl_vucc_grids).""; - $table .= "".$iota.""; - $table .= "QSO Record: ".$status[0].""; + $table .= "".$row['station_callsign'].""; + $table .= "".$row['time_on'].""; + $table .= "".$row['call'].""; + $table .= "".$row['mode'].""; + $table .= "".$row['qsl_rcvd'].""; + $table .= "".$row['qsl_date'].""; + $table .= "".$row['state'].""; + $table .= "".$row['gridsquare'].""; + $table .= "".$row['iota'].""; + $table .= "QSO Record: ".$row['status'].""; $table .= "LoTW Record: ".$lotw_status.""; $table .= ""; } else { $table .= ""; - $table .= "".$record['station_callsign'].""; - $table .= "".$time_on.""; - $table .= "".$record['call'].""; - $table .= "".$record['mode'].""; - $table .= "".$record['qsl_rcvd'].""; + $table .= "".$row['station_callsign'].""; + $table .= "".$row['time_on'].""; + $table .= "".$row['call'].""; + $table .= "".$row['mode'].""; + $table .= "".$row['qsl_rcvd'].""; $table .= ""; $table .= ""; $table .= ""; $table .= ""; - $table .= "QSO Record: ".$status[0].""; + $table .= "QSO Record: ".$row['status'].""; $table .= ""; $table .= ""; } diff --git a/application/migrations/247_add_lotw_performance_indexes.php b/application/migrations/247_add_lotw_performance_indexes.php new file mode 100644 index 000000000..4cd471862 --- /dev/null +++ b/application/migrations/247_add_lotw_performance_indexes.php @@ -0,0 +1,78 @@ +db->db_debug = false; + + $table_name = $this->config->item('table_name'); + + // Add index on COL_LOTW_QSLRDATE for lotw_last_qsl_date() query + // Dramatically improves performance when finding the last LOTW confirmation date + $lotw_qslrdate_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qslrdate'")->num_rows(); + + if ($lotw_qslrdate_index_exists == 0) { + $sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_qslrdate` (`COL_LOTW_QSLRDATE`)"; + $this->db->query($sql); + } + + // Add composite index for get_lotw_qsos_to_upload() query + // This index covers the WHERE and ORDER BY clauses for efficient LOTW upload preparation + // Format: (station_id, COL_LOTW_QSL_SENT, COL_TIME_ON) + $lotw_upload_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qsos_to_upload'")->num_rows(); + + if ($lotw_upload_index_exists == 0) { + $sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_qsos_to_upload` (`station_id`, `COL_LOTW_QSL_SENT`, `COL_TIME_ON`)"; + $this->db->query($sql); + } + + // Add composite index for LOTW confirmation matching in batch downloads + // Speeds up import_check() and lotw_update_batch() lookups + // Format: (COL_TIME_ON, COL_CALL, COL_BAND, COL_STATION_CALLSIGN) + $lotw_match_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_confirmation_match'")->num_rows(); + + if ($lotw_match_index_exists == 0) { + $sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_confirmation_match` (`COL_TIME_ON`, `COL_CALL`, `COL_BAND`, `COL_STATION_CALLSIGN`)"; + $this->db->query($sql); + } + + $this->db->db_debug = true; + } + + public function down() + { + $this->db->db_debug = false; + + $table_name = $this->config->item('table_name'); + + // Drop the LOTW QSLRDATE index if it exists + $lotw_qslrdate_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qslrdate'")->num_rows(); + + if ($lotw_qslrdate_index_exists > 0) { + $sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_qslrdate`"; + $this->db->query($sql); + } + + // Drop the LOTW upload index if it exists + $lotw_upload_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qsos_to_upload'")->num_rows(); + + if ($lotw_upload_index_exists > 0) { + $sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_qsos_to_upload`"; + $this->db->query($sql); + } + + // Drop the LOTW confirmation match index if it exists + $lotw_match_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_confirmation_match'")->num_rows(); + + if ($lotw_match_index_exists > 0) { + $sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_confirmation_match`"; + $this->db->query($sql); + } + + $this->db->db_debug = true; + } +} diff --git a/application/models/Logbook_model.php b/application/models/Logbook_model.php index ab0b625dc..5237aaf68 100755 --- a/application/models/Logbook_model.php +++ b/application/models/Logbook_model.php @@ -3504,6 +3504,182 @@ class Logbook_model extends CI_Model return "Updated"; } + /* + * Batch update multiple QSOs from LoTW confirmation downloads + * Replaces N individual lotw_update() calls with batch operations + * Provides massive performance improvement for large LOTW downloads + * + * @param array $records Array of LOTW confirmation records with keys: + * datetime, callsign, band, qsl_date, qsl_status, + * state, qsl_gridsquare, qsl_vucc_grids, iota, + * cnty, cqz, ituz, station_callsign + * @return array Statistics array with 'updated', 'gridsquare_updated', 'errors' counts + */ + function lotw_update_batch($records) + { + if (empty($records) || !is_array($records)) { + log_message('debug', 'LoTW batch update: No records provided'); + return array('updated' => 0, 'gridsquare_updated' => 0, 'errors' => 0); + } + + $record_count = count($records); + log_message('info', "LoTW batch update: Processing {$record_count} confirmation records"); + + $table_name = $this->config->item('table_name'); + + // Step 1: Build WHERE conditions to find all matching QSOs and their current upload status + $match_conditions = array(); + foreach ($records as $idx => $record) { + $match_conditions[] = sprintf( + "(date_format(COL_TIME_ON, '%%Y-%%m-%%d %%H:%%i') = '%s' AND COL_CALL = '%s' AND COL_BAND = '%s' AND COL_STATION_CALLSIGN = '%s')", + $this->db->escape_str($record['datetime']), + $this->db->escape_str($record['callsign']), + $this->db->escape_str($record['band']), + $this->db->escape_str($record['station_callsign']) + ); + } + + // Step 2: Get all matching QSOs with their current upload status and gridsquare + $sql = "SELECT COL_PRIMARY_KEY, + date_format(COL_TIME_ON, '%Y-%m-%d %H:%i') as fmt_time, + COL_CALL, COL_BAND, COL_STATION_CALLSIGN, + COL_CLUBLOG_QSO_UPLOAD_STATUS as CL_STATE, + COL_QRZCOM_QSO_UPLOAD_STATUS as QRZ_STATE, + COL_LOTW_QSL_RCVD, + station_profile.station_gridsquare, + station_profile.station_id + FROM {$table_name} + LEFT JOIN station_profile ON {$table_name}.station_id = station_profile.station_id + WHERE (" . implode(' OR ', $match_conditions) . ")"; + + $query = $this->db->query($sql); + + if ($query->num_rows() == 0) { + log_message('warning', 'LoTW batch update: No matching QSOs found in database'); + return array('updated' => 0, 'gridsquare_updated' => 0, 'errors' => 0); + } + + // Step 3: Build lookup map and batch update array + $qso_map = array(); + foreach ($query->result() as $qso) { + $key = $qso->fmt_time . '|' . $qso->COL_CALL . '|' . $qso->COL_BAND . '|' . $qso->COL_STATION_CALLSIGN; + $qso_map[$key] = $qso; + } + + $batch_updates = array(); + $gridsquare_updates = array(); + + foreach ($records as $record) { + $key = $record['datetime'] . '|' . $record['callsign'] . '|' . $record['band'] . '|' . $record['station_callsign']; + + if (!isset($qso_map[$key])) { + continue; // QSO not found in database + } + + $qso = $qso_map[$key]; + + // Skip if already updated with this exact status + if ($qso->COL_LOTW_QSL_RCVD == $record['qsl_status']) { + continue; + } + + $update_data = array( + 'COL_PRIMARY_KEY' => $qso->COL_PRIMARY_KEY, + 'COL_LOTW_QSLRDATE' => $record['qsl_date'], + 'COL_LOTW_QSL_RCVD' => $record['qsl_status'], + 'COL_LOTW_QSL_SENT' => 'Y' + ); + + // Add optional fields + if (!empty($record['state'])) { + $update_data['COL_STATE'] = $record['state']; + } + if (!empty($record['iota'])) { + $update_data['COL_IOTA'] = $record['iota']; + } + if (!empty($record['cnty'])) { + $update_data['COL_CNTY'] = $record['cnty']; + } + if (!empty($record['cqz'])) { + $update_data['COL_CQZ'] = $record['cqz']; + } + if (!empty($record['ituz'])) { + $update_data['COL_ITUZ'] = $record['ituz']; + } + + // Mark for re-upload to QRZ/ClubLog if already uploaded + if ($qso->QRZ_STATE == 'Y') { + $update_data['COL_QRZCOM_QSO_UPLOAD_STATUS'] = 'M'; + } + if ($qso->CL_STATE == 'Y') { + $update_data['COL_CLUBLOG_QSO_UPLOAD_STATUS'] = 'M'; + } + + $batch_updates[] = $update_data; + + // Handle gridsquare updates separately (need distance calculation) + if (!empty($record['qsl_gridsquare']) || !empty($record['qsl_vucc_grids'])) { + $gridsquare_updates[] = array( + 'COL_PRIMARY_KEY' => $qso->COL_PRIMARY_KEY, + 'station_gridsquare' => $qso->station_gridsquare, + 'qsl_gridsquare' => $record['qsl_gridsquare'] ?? '', + 'qsl_vucc_grids' => $record['qsl_vucc_grids'] ?? '' + ); + } + } + + // Step 4: Execute batch update + $updated_count = 0; + if (!empty($batch_updates)) { + $this->db->update_batch($table_name, $batch_updates, 'COL_PRIMARY_KEY'); + $updated_count = $this->db->affected_rows(); + log_message('info', "LoTW batch update: Updated {$updated_count} QSO records"); + } + + // Step 5: Handle gridsquare updates with distance calculation + $gridsquare_count = 0; + if (!empty($gridsquare_updates)) { + if (!$this->load->is_loaded('Qra')) { + $this->load->library('Qra'); + } + + $grid_batch = array(); + foreach ($gridsquare_updates as $grid_update) { + $data = array('COL_PRIMARY_KEY' => $grid_update['COL_PRIMARY_KEY']); + + if (!empty($grid_update['qsl_gridsquare'])) { + $data['COL_GRIDSQUARE'] = $grid_update['qsl_gridsquare']; + $data['COL_DISTANCE'] = $this->qra->distance( + $grid_update['station_gridsquare'], + $grid_update['qsl_gridsquare'], + 'K' + ); + } elseif (!empty($grid_update['qsl_vucc_grids'])) { + $data['COL_VUCC_GRIDS'] = $grid_update['qsl_vucc_grids']; + $data['COL_DISTANCE'] = $this->qra->distance( + $grid_update['station_gridsquare'], + $grid_update['qsl_vucc_grids'], + 'K' + ); + } + + $grid_batch[] = $data; + } + + if (!empty($grid_batch)) { + $this->db->update_batch($table_name, $grid_batch, 'COL_PRIMARY_KEY'); + $gridsquare_count = $this->db->affected_rows(); + log_message('info', "LoTW batch update: Updated {$gridsquare_count} gridsquare/distance records"); + } + } + + return array( + 'updated' => $updated_count, + 'gridsquare_updated' => $gridsquare_count, + 'errors' => 0 + ); + } + function qrz_last_qsl_date($user_id) { $sql = "SELECT date_format(MAX(COALESCE(COL_QRZCOM_QSO_DOWNLOAD_DATE, str_to_date('1900-01-01','%Y-%m-%d'))),'%Y-%m-%d') MAXDATE diff --git a/application/models/LotwCert.php b/application/models/LotwCert.php index 0c97fc699..7d49ac0c9 100644 --- a/application/models/LotwCert.php +++ b/application/models/LotwCert.php @@ -78,26 +78,28 @@ class LotwCert extends CI_Model { } function toggle_archive_certificate($user_id, $lotw_cert_id) { - // First get current archive status + // Use single UPDATE with calculated toggle instead of SELECT+UPDATE + // archived = 1 - archived toggles between 0 and 1 + $this->db->set('archived', '1 - archived', FALSE); + $this->db->where('lotw_cert_id', $lotw_cert_id); + $this->db->where('user_id', $user_id); + $this->db->update('lotw_certs'); + + if ($this->db->affected_rows() == 0) { + return false; + } + + // Get final status to return $this->db->select('archived'); $this->db->where('lotw_cert_id', $lotw_cert_id); $this->db->where('user_id', $user_id); $query = $this->db->get('lotw_certs'); - if($query->num_rows() == 0) { - return false; + if($query->num_rows() > 0) { + return array('archived' => $query->row()->archived); } - $current_status = $query->row()->archived; - $new_status = $current_status ? 0 : 1; - - // Update the archive status - $data = array('archived' => $new_status); - $this->db->where('lotw_cert_id', $lotw_cert_id); - $this->db->where('user_id', $user_id); - $this->db->update('lotw_certs', $data); - - return array('archived' => $new_status); + return false; } function last_upload($certID) {