mirror of
https://github.com/magicbug/Cloudlog
synced 2026-08-13 17:49:35 -04:00
9fe5b808a5
Changed the import logic to deduplicate LoTW users by callsign, keeping only the latest timestamp for each callsign. Updated batch insertion to process only unique callsigns and improved reporting to show both total records read and unique callsigns inserted.
640 lines
21 KiB
PHP
640 lines
21 KiB
PHP
<?php
|
|
class Update extends CI_Controller {
|
|
|
|
/*
|
|
Controls Updating Elements of Cloudlog
|
|
Functions:
|
|
dxcc - imports the latest clublog cty.xml data
|
|
lotw_users - imports lotw users
|
|
*/
|
|
|
|
public function index()
|
|
{
|
|
$data['page_title'] = "Updates";
|
|
$this->load->view('interface_assets/header', $data);
|
|
$this->load->view('update/index');
|
|
$this->load->view('interface_assets/footer');
|
|
|
|
}
|
|
|
|
/*
|
|
* Create a path to a file in the updates folder, respecting the datadir
|
|
* configuration option.
|
|
*/
|
|
private function make_update_path($path) {
|
|
$path = "updates/" . $path;
|
|
$datadir = $this->config->item('datadir');
|
|
if(!$datadir) {
|
|
return $path;
|
|
}
|
|
return $datadir . "/" . $path;
|
|
}
|
|
|
|
/*
|
|
* Load the dxcc entities
|
|
*/
|
|
public function dxcc_entities() {
|
|
// Load Database connectors
|
|
$this->load->model('dxcc_entities');
|
|
|
|
// Load the cty file
|
|
$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
|
|
|
|
//$xml_data->entities->entity->count();
|
|
|
|
$count = 0;
|
|
foreach ($xml_data->entities->entity as $entity) {
|
|
$startinfo = strtotime($entity->start);
|
|
$endinfo = strtotime($entity->end);
|
|
|
|
$start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
|
|
$end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
|
|
|
|
if(!$entity->cqz) {
|
|
$data = array(
|
|
'prefix' => (string) $entity->call,
|
|
'name' => (string) $entity->entity,
|
|
);
|
|
} else {
|
|
$data = array(
|
|
'adif' => (int) $entity->adif,
|
|
'name' => (string) $entity->name,
|
|
'prefix' => (string) $entity->prefix,
|
|
'ituz' => (float) $entity->ituz,
|
|
'cqz' => (int) $entity->cqz,
|
|
'cont' => (string) $entity->cont,
|
|
'long' => (float) $entity->long,
|
|
'lat' => (float) $entity->lat,
|
|
'start' => $start_date,
|
|
'end' => $end_date,
|
|
);
|
|
}
|
|
|
|
$this->db->insert('dxcc_entities', $data);
|
|
$count += 1;
|
|
if ($count % 10 == 0)
|
|
$this->update_status();
|
|
}
|
|
|
|
$this->update_status();
|
|
return $count;
|
|
}
|
|
|
|
/*
|
|
* Load the dxcc exceptions
|
|
*/
|
|
public function dxcc_exceptions() {
|
|
// Load Database connectors
|
|
$this->load->model('dxcc_exceptions');
|
|
// Load the cty file
|
|
$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
|
|
|
|
$count = 0;
|
|
foreach ($xml_data->exceptions->exception as $record) {
|
|
$startinfo = strtotime($record->start);
|
|
$endinfo = strtotime($record->end);
|
|
|
|
$start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
|
|
$end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
|
|
|
|
$data = array(
|
|
'record' => (int) $record->attributes()->record,
|
|
'call' => (string) $record->call,
|
|
'entity' => (string) $record->entity,
|
|
'adif' => (int) $record->adif,
|
|
'cqz' => (int) $record->cqz,
|
|
'cont' => (string) $record->cont,
|
|
'long' => (float) $record->long,
|
|
'lat' => (float) $record->lat,
|
|
'start' => $start_date,
|
|
'end' => $end_date,
|
|
);
|
|
|
|
$this->db->insert('dxcc_exceptions', $data);
|
|
$count += 1;
|
|
if ($count % 10 == 0)
|
|
$this->update_status();
|
|
}
|
|
|
|
$this->update_status();
|
|
return $count;
|
|
}
|
|
|
|
/*
|
|
* Load the dxcc prefixes
|
|
*/
|
|
public function dxcc_prefixes() {
|
|
// Load Database connectors
|
|
$this->load->model('dxcc_prefixes');
|
|
// Load the cty file
|
|
$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
|
|
|
|
$count = 0;
|
|
foreach ($xml_data->prefixes->prefix as $record) {
|
|
$startinfo = strtotime($record->start);
|
|
$endinfo = strtotime($record->end);
|
|
|
|
$start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
|
|
$end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
|
|
|
|
$data = array(
|
|
'record' => (int) $record->attributes()->record,
|
|
'call' => (string) $record->call,
|
|
'entity' => (string) $record->entity,
|
|
'adif' => (int) $record->adif,
|
|
'cqz' => (int) $record->cqz,
|
|
'cont' => (string) $record->cont,
|
|
'long' => (float) $record->long,
|
|
'lat' => (float) $record->lat,
|
|
'start' => $start_date,
|
|
'end' => $end_date,
|
|
);
|
|
|
|
$this->db->insert('dxcc_prefixes', $data);
|
|
$count += 1;
|
|
if ($count % 10 == 0)
|
|
$this->update_status();
|
|
}
|
|
|
|
//print("$count prefixes processed");
|
|
$this->update_status();
|
|
return $count;
|
|
}
|
|
|
|
// Updates the DXCC & Exceptions from the Club Log Cty.xml file.
|
|
public function dxcc() {
|
|
$this->update_status("Downloading file");
|
|
|
|
// give it 10 minutes...
|
|
set_time_limit(600);
|
|
|
|
// Load Migration data if any.
|
|
$this->load->library('migration');
|
|
$this->fix_migrations();
|
|
$this->migration->latest();
|
|
|
|
// Download latest file.
|
|
$url = "https://cdn.clublog.org/cty.php?api=a11c3235cd74b88212ce726857056939d52372bd";
|
|
|
|
$gz = gzopen($url, 'r');
|
|
if ($gz === FALSE) {
|
|
// If the download from clublog.org fails, try cloudlog.org CDN.
|
|
$url = "https://cdn.cloudlog.org/clublogxml.gz";
|
|
$gz = gzopen($url, 'r');
|
|
|
|
// Log failure to log file
|
|
log_message('info', 'Failed to download cty.xml from clublog.org, trying cloudlog.org CDN');
|
|
|
|
if ($gz === FALSE) {
|
|
$this->update_status("FAILED: Could not download from clublog.org or cloudlog.org");
|
|
log_message('error', 'FAILED: Could not download exceptions from clublog.org or cloudlog.org');
|
|
return;
|
|
}
|
|
}
|
|
|
|
$data = "";
|
|
while (!gzeof($gz)) {
|
|
$data .= gzgetc($gz);
|
|
}
|
|
gzclose($gz);
|
|
|
|
if (file_put_contents($this->make_update_path("cty.xml"), $data) === FALSE) {
|
|
$this->update_status("FAILED: Could not write to cty.xml file");
|
|
return;
|
|
}
|
|
|
|
// Clear the tables, ready for new data
|
|
$this->db->empty_table("dxcc_entities");
|
|
$this->db->empty_table("dxcc_exceptions");
|
|
$this->db->empty_table("dxcc_prefixes");
|
|
$this->update_status();
|
|
|
|
// Parse the three sections of the file and update the tables
|
|
$this->db->trans_start();
|
|
$this->dxcc_entities();
|
|
$this->dxcc_exceptions();
|
|
$this->dxcc_prefixes();
|
|
$this->db->trans_complete();
|
|
|
|
$this->update_status("DONE");
|
|
}
|
|
|
|
public function get_status() {
|
|
$status_file = $this->make_update_path("status.html");
|
|
if (file_exists($status_file)) {
|
|
$content = file_get_contents($status_file);
|
|
echo $content;
|
|
} else {
|
|
echo "No status available";
|
|
}
|
|
}
|
|
|
|
public function update_status($done=""){
|
|
|
|
if ($done != "Downloading file"){
|
|
// Check that everything is done?
|
|
if ($done == ""){
|
|
$done = "Updating...";
|
|
}
|
|
$html = $done."<br/>";
|
|
$html .= "Dxcc Entities: ".$this->db->count_all('dxcc_entities')."<br/>";
|
|
$html .= "Dxcc Exceptions: ".$this->db->count_all('dxcc_exceptions')."<br/>";
|
|
$html .= "Dxcc Prefixes: ".$this->db->count_all('dxcc_prefixes')."<br/>";
|
|
}else{
|
|
$html = $done."....<br/>";
|
|
}
|
|
|
|
$status_file = $this->make_update_path("status.html");
|
|
if (file_put_contents($status_file, $html) === FALSE) {
|
|
log_message('error', 'Failed to write status file: ' . $status_file);
|
|
// Try to create the directory if it doesn't exist
|
|
$dir = dirname($status_file);
|
|
if (!is_dir($dir)) {
|
|
mkdir($dir, 0755, true);
|
|
file_put_contents($status_file, $html);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
private function fix_migrations(){
|
|
$res = $this->db->query("select version from migrations");
|
|
if ($res->num_rows() >0){
|
|
$row = $res->row();
|
|
$version = $row->version;
|
|
|
|
if ($version < 7){
|
|
$this->db->query("update migrations set version=7");
|
|
}
|
|
}
|
|
}
|
|
|
|
public function check_missing_dxcc($all = false){
|
|
$this->load->model('logbook_model');
|
|
$this->logbook_model->check_missing_dxcc_id($all);
|
|
|
|
}
|
|
|
|
public function check_missing_continent() {
|
|
$this->load->model('logbook_model');
|
|
$this->logbook_model->check_missing_continent();
|
|
}
|
|
|
|
public function update_distances() {
|
|
$this->load->model('logbook_model');
|
|
$this->logbook_model->update_distances();
|
|
}
|
|
|
|
public function check_missing_grid($all = false){
|
|
$this->load->model('logbook_model');
|
|
$this->logbook_model->check_missing_grid_id($all);
|
|
}
|
|
|
|
public function update_clublog_scp() {
|
|
$strFile = $this->make_update_path("clublog_scp.txt");
|
|
$url = "https://cdn.clublog.org/clublog.scp.gz";
|
|
set_time_limit(300);
|
|
echo "Downloading Club Log SCP file...<br>";
|
|
$gz = gzopen($url, 'r');
|
|
if ($gz)
|
|
{
|
|
$data = "";
|
|
while (!gzeof($gz)) {
|
|
$data .= gzgetc($gz);
|
|
}
|
|
gzclose($gz);
|
|
if (file_put_contents($strFile, $data) !== FALSE)
|
|
{
|
|
$nCount = count(file($strFile));
|
|
if ($nCount > 0)
|
|
{
|
|
echo "DONE: " . number_format($nCount) . " callsigns loaded";
|
|
} else {
|
|
echo "FAILED: Empty file";
|
|
}
|
|
} else {
|
|
echo "FAILED: Could not write to Club Log SCP file";
|
|
}
|
|
} else {
|
|
echo "FAILED: Could not connect to Club Log";
|
|
}
|
|
}
|
|
|
|
public function download_lotw_users() {
|
|
$contents = file_get_contents('https://lotw.arrl.org/lotw-user-activity.csv', true);
|
|
|
|
if($contents === FALSE) {
|
|
echo "Something went wrong with fetching the LoTW users file.";
|
|
} else {
|
|
$file = './updates/lotw_users.csv';
|
|
|
|
if (file_put_contents($file, $contents) !== FALSE) { // Save our content to the file.
|
|
echo "LoTW User Data Saved.";
|
|
} else {
|
|
echo "FAILED: Could not write to LoTW users file";
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
public function lotw_users() {
|
|
$mtime = microtime();
|
|
$mtime = explode(" ",$mtime);
|
|
$mtime = $mtime[1] + $mtime[0];
|
|
$starttime = $mtime;
|
|
|
|
$file = 'https://lotw.arrl.org/lotw-user-activity.csv';
|
|
|
|
// First, try to download and validate the file
|
|
$context = stream_context_create([
|
|
'http' => [
|
|
'timeout' => 60, // 60 second timeout
|
|
'user_agent' => 'Cloudlog LoTW User Updater'
|
|
]
|
|
]);
|
|
|
|
$handle = fopen($file, "r", false, $context);
|
|
if ($handle === FALSE) {
|
|
echo "FAILED: Could not connect to LoTW server";
|
|
log_message('error', 'Failed to connect to LoTW user activity CSV');
|
|
return;
|
|
}
|
|
|
|
// Read and validate the first few lines to ensure we have valid data
|
|
$line_count = 0;
|
|
$sample_data = array();
|
|
while ($line_count < 5 && ($line = fgetcsv($handle, 1000, ",")) !== FALSE) {
|
|
if ($line[0]) { // Skip empty lines
|
|
$sample_data[] = $line;
|
|
$line_count++;
|
|
}
|
|
}
|
|
|
|
// Validate we have enough sample data
|
|
if ($line_count < 2) {
|
|
fclose($handle);
|
|
echo "FAILED: LoTW file appears to be empty or invalid";
|
|
log_message('error', 'LoTW user activity CSV appears empty or invalid');
|
|
return;
|
|
}
|
|
|
|
// Reset file pointer to beginning
|
|
fclose($handle);
|
|
$handle = fopen($file, "r", false, $context);
|
|
if ($handle === FALSE) {
|
|
echo "FAILED: Could not reopen LoTW file";
|
|
return;
|
|
}
|
|
|
|
// Only truncate table AFTER we've validated the remote file
|
|
$this->db->query("TRUNCATE TABLE lotw_users");
|
|
|
|
$i = 0; // raw rows read
|
|
$batch_count = 0;
|
|
// Use a map to deduplicate by callsign and keep the latest timestamp
|
|
$lotw_map = array();
|
|
$batch_size = 500; // Smaller batch size for better performance and memory usage
|
|
|
|
// Skip CSV header row
|
|
$data = fgetcsv($handle,1000,",");
|
|
|
|
while (($data = fgetcsv($handle,1000,",")) !== FALSE) {
|
|
if ($data[0] && isset($data[1]) && isset($data[2])) {
|
|
// Normalize callsign to uppercase before validation
|
|
$callsign = strtoupper($data[0]);
|
|
// Validate callsign format (basic check)
|
|
if (preg_match('/^[A-Z0-9\/]+$/', $callsign)) {
|
|
$i++;
|
|
// Compose timestamp string and compare; keep the latest per callsign
|
|
$ts = $data[1] . ' ' . $data[2];
|
|
// If we haven't seen this callsign, or this row is newer, store it
|
|
if (!isset($lotw_map[$callsign]) || strtotime($ts) > strtotime($lotw_map[$callsign])) {
|
|
$lotw_map[$callsign] = $ts;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
fclose($handle);
|
|
|
|
// Insert deduplicated records in batches
|
|
if (!empty($lotw_map)) {
|
|
$lotwdata = array();
|
|
foreach ($lotw_map as $cs => $lu) {
|
|
$lotwdata[] = array('callsign' => $cs, 'lastupload' => $lu);
|
|
if (count($lotwdata) >= $batch_size) {
|
|
if (!$this->db->insert_batch('lotw_users', $lotwdata)) {
|
|
echo "FAILED: Database error during batch insert";
|
|
log_message('error', 'Database error during LoTW batch insert while inserting deduped map');
|
|
return;
|
|
}
|
|
$batch_count++;
|
|
$lotwdata = array();
|
|
}
|
|
}
|
|
// Final batch
|
|
if (!empty($lotwdata)) {
|
|
if (!$this->db->insert_batch('lotw_users', $lotwdata)) {
|
|
echo "FAILED: Database error during final batch insert";
|
|
log_message('error', 'Database error during LoTW final batch insert while inserting deduped map');
|
|
return;
|
|
}
|
|
$batch_count++;
|
|
}
|
|
}
|
|
|
|
// Verify we actually imported data
|
|
$final_count = $this->db->count_all('lotw_users');
|
|
if ($final_count == 0) {
|
|
echo "FAILED: No records were imported - possible data corruption";
|
|
log_message('error', 'LoTW import completed but no records in database');
|
|
return;
|
|
}
|
|
|
|
if ($i < 50000) { // Sanity check - LoTW should have at least 50k users
|
|
echo "WARNING: Only imported " . $i . " records - this seems unusually low";
|
|
log_message('warning', 'LoTW import record count unusually low: ' . $i);
|
|
}
|
|
|
|
$mtime = microtime();
|
|
$mtime = explode(" ",$mtime);
|
|
$mtime = $mtime[1] + $mtime[0];
|
|
$endtime = $mtime;
|
|
$totaltime = ($endtime - $starttime);
|
|
echo "This page was created in ".$totaltime." seconds <br />";
|
|
echo "Records read: " . $i . " <br/>";
|
|
echo "Unique callsigns inserted: " . $final_count . " <br/>";
|
|
}
|
|
|
|
public function lotw_check() {
|
|
$f = fopen('./updates/lotw_users.csv', "r");
|
|
$result = false;
|
|
while ($row = fgetcsv($f)) {
|
|
if ($row[0] == '2M0SQL/MM') {
|
|
$result = $row[0];
|
|
echo "found";
|
|
break;
|
|
}
|
|
}
|
|
fclose($f);
|
|
}
|
|
|
|
/*
|
|
* Used for autoupdating the DOK file which is used in the QSO entry dialog for autocompletion.
|
|
*/
|
|
public function update_dok() {
|
|
$contents = file_get_contents('https://www.df2et.de/cqrlog/dok_and_sdok.txt', true);
|
|
|
|
if($contents === FALSE) {
|
|
echo "Something went wrong with fetching the DOK file.";
|
|
} else {
|
|
$file = './assets/json/dok.txt';
|
|
|
|
if (file_put_contents($file, $contents) !== FALSE) { // Save our content to the file.
|
|
$nCount = count(file($file));
|
|
if ($nCount > 0)
|
|
{
|
|
echo "DONE: " . number_format($nCount) . " DOKs and SDOKs saved";
|
|
} else {
|
|
echo"FAILED: Empty file";
|
|
}
|
|
} else {
|
|
echo"FAILED: Could not write to dok.txt file";
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Used for autoupdating the SOTA file which is used in the QSO entry dialog for autocompletion.
|
|
*/
|
|
public function update_sota() {
|
|
$csvfile = 'https://www.sotadata.org.uk/summitslist.csv';
|
|
|
|
$sotafile = './assets/json/sota.txt';
|
|
|
|
$csvhandle = fopen($csvfile,"r");
|
|
if ($csvhandle === FALSE) {
|
|
echo "Something went wrong with fetching the SOTA file";
|
|
return;
|
|
}
|
|
|
|
$data = fgetcsv($csvhandle,1000,","); // Skip line we are not interested in
|
|
$data = fgetcsv($csvhandle,1000,","); // Skip line we are not interested in
|
|
$data = fgetcsv($csvhandle,1000,",");
|
|
$sotafilehandle = fopen($sotafile, 'w');
|
|
|
|
if ($sotafilehandle === FALSE) {
|
|
echo"FAILED: Could not write to sota.txt file";
|
|
return;
|
|
}
|
|
|
|
$nCount = 0;
|
|
do {
|
|
if ($data[0]) {
|
|
fwrite($sotafilehandle, $data[0].PHP_EOL);
|
|
$nCount++;
|
|
}
|
|
} while ($data = fgetcsv($csvhandle,1000,","));
|
|
|
|
fclose($csvhandle);
|
|
fclose($sotafilehandle);
|
|
|
|
if ($nCount > 0)
|
|
{
|
|
echo "DONE: " . number_format($nCount) . " SOTA's saved";
|
|
} else {
|
|
echo"FAILED: Empty file";
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Pulls the WWFF directory for autocompletion in QSO dialogs
|
|
*/
|
|
public function update_wwff() {
|
|
$csvfile = 'https://wwff.co/wwff-data/wwff_directory.csv';
|
|
|
|
$wwfffile = './assets/json/wwff.txt';
|
|
|
|
$ch = curl_init();
|
|
curl_setopt($ch, CURLOPT_URL, $csvfile);
|
|
curl_setopt($ch, CURLOPT_HEADER, false);
|
|
curl_setopt($ch, CURLOPT_USERAGENT, 'Cloudlog Updater');
|
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
$csv = curl_exec($ch);
|
|
curl_close($ch);
|
|
if ($csv === FALSE) {
|
|
echo "Something went wrong with fetching the WWFF file";
|
|
return;
|
|
}
|
|
|
|
$wwfffilehandle = fopen($wwfffile, 'w');
|
|
if ($wwfffilehandle === FALSE) {
|
|
echo"FAILED: Could not write to wwff.txt file";
|
|
return;
|
|
}
|
|
|
|
$data = str_getcsv($csv,"\n");
|
|
$nCount = 0;
|
|
foreach ($data as $idx => $row) {
|
|
if ($idx == 0) continue; // Skip line we are not interested in
|
|
$row = str_getcsv($row, ',');
|
|
if ($row[0]) {
|
|
fwrite($wwfffilehandle, $row[0].PHP_EOL);
|
|
$nCount++;
|
|
}
|
|
}
|
|
|
|
fclose($wwfffilehandle);
|
|
|
|
if ($nCount > 0)
|
|
{
|
|
echo "DONE: " . number_format($nCount) . " WWFF's saved";
|
|
} else {
|
|
echo"FAILED: Empty file";
|
|
}
|
|
}
|
|
|
|
public function update_pota() {
|
|
$csvfile = 'https://pota.app/all_parks.csv';
|
|
|
|
$potafile = './assets/json/pota.txt';
|
|
|
|
$ch = curl_init();
|
|
curl_setopt($ch, CURLOPT_URL, $csvfile);
|
|
curl_setopt($ch, CURLOPT_HEADER, false);
|
|
curl_setopt($ch, CURLOPT_USERAGENT, 'Cloudlog Updater');
|
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
$csv = curl_exec($ch);
|
|
curl_close($ch);
|
|
if ($csv === FALSE) {
|
|
echo "Something went wrong with fetching the POTA file";
|
|
return;
|
|
}
|
|
|
|
$potafilehandle = fopen($potafile, 'w');
|
|
if ($potafilehandle === FALSE) {
|
|
echo"FAILED: Could not write to pota.txt file";
|
|
return;
|
|
}
|
|
$data = str_getcsv($csv,"\n");
|
|
$nCount = 0;
|
|
foreach ($data as $idx => $row) {
|
|
if ($idx == 0) continue; // Skip line we are not interested in
|
|
$row = str_getcsv($row, ',');
|
|
if ($row[0]) {
|
|
fwrite($potafilehandle, $row[0].PHP_EOL);
|
|
$nCount++;
|
|
}
|
|
}
|
|
|
|
fclose($potafilehandle);
|
|
|
|
if ($nCount > 0)
|
|
{
|
|
echo "DONE: " . number_format($nCount) . " POTA's saved";
|
|
} else {
|
|
echo"FAILED: Empty file";
|
|
}
|
|
}
|
|
|
|
}
|
|
?>
|