<!doctype html>
<html>
<head>
<title>Geoid .grd file parser</title>
<style type="text/css">
body > div {
height: 1px;
}
div div {
display: inline-block;
vertical-align: top;
height: 1px;
width: 1px;
}
</style>
<script type="text/javascript">
window.onload = function () {
var p = document.getElementById('stats'), loadedtime = (new Date()).getTime(), showloading;
if( p && document.getElementsByTagName('canvas').length ) {
p.appendChild( document.createTextNode( ' Loading took ' + ( ( loadedtime - startloading ) / 1000 ) + ' seconds.' ) );
paintCanvas(0);
p.appendChild( document.createTextNode( ' Rendering took ' + ( ( (new Date()).getTime() - loadedtime ) / 1000 ) + ' seconds.' ) );
}
var showloading = document.getElementById('showloading');
if( showloading ) {
showloading.parentNode.removeChild(showloading);
}
};
var startloading = (new Date()).getTime();
</script>
</head>
<body>
<h1>Geoid .grd file parser</h1>
<?php
$override_file_name = ''; //hardcode a file to parse instead
$canbatch = true; //can your database handle batching 1000 results at a time, to improve performance, uses a little more memory
function put_data_in_database( $recordindex, $latindex, $longindex, $latitude, $longitude, $latstepdp, $longstepdp, $data, $identifier, $final = false ) {
$latitude = round( $latitude, $latstepdp );
$longitude = round( $longitude, $longstepdp );
global $dbhandle, $databasecount, $canbatch, $batchdata, $filehandle;
/*
THIS IS WHERE YOU ADD THE DATA TO THE DATABASE
$latindex = point latitude index (latitude row with lowest index = 0)
$longindex = point longitude index (longitude column with lowest index = 0)
$latitude = latitude rounded to the latitude precision given in the header
$longitude = longitude rounded to the longitude precision given in the header
$data = the actual geoid data for this data point
$identifier = identification name for the geoid set in the database
*/
if( !$final ) {
//the database record to be added
$batchdata[] = "( $recordindex, $latindex, $longindex, $latitude, $longitude, $data )";
$databasecount++;
}
if( !$canbatch || !( $databasecount % 1000 ) || $final ) {
if( count($batchdata) ) {
//the rest of the database query
database_query_command( 'INSERT INTO `'.addslashes($_POST['grdname']).'` VALUES ' . implode( ',', $batchdata ), $dbhandle );
if( $error = database_error_command() ) {
print '<p>Error: '.htmlspecialchars($error).'</p>';
fclose($filehandle);
exit;
}
$batchdata = array();
}
}
if( !( $databasecount % 1000 ) ) {
print '<script type="text/javascript">document.getElementById("showloading").textContent = "' . $databasecount . ' items added to database...";</script>';
ob_flush();
flush();
}
}
$dbhandle = null;
$databasecount = 0;
$maxsizereadable = ini_get('upload_max_filesize');
$batchdata = array();
if( $override_file_name || ( isset($_FILES['grdfile']) && $_FILES['grdfile']['error'] == UPLOAD_ERR_OK && $_FILES['grdfile']['tmp_name'] ) ) {
//work out how much memory to allocate - these values are based on a 9.17 MB file, EGM96 ww15mgh.grd
if( $_POST['dataaction'] == 'render' || $_POST['dataaction'] == 'renderaddlowmem' ) {
//uses 11 times the size of the file
ini_set('memory_limit','128M');
} elseif( $_POST['dataaction'] == 'renderadd' || $_POST['dataaction'] == 'add' ) {
//uses 47 times the size of the file
ini_set('memory_limit','512M');
}
if( $_POST['dataaction'] == 'renderadd' || $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'add' || $_POST['dataaction'] == 'addlowmem' ) {
//it takes a lot of time to add a lot of database entries - adjust it here
set_time_limit( 60 * 60 );
if( $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'addlowmem' ) {
/*
THIS IS WHERE YOU CONNECT TO THE DATABASE (CONNECT FURTHER DOWN IF USING renderadd OR add)
$dbhandle = your database connection handle
Set primary keys according to the query method you will use, indexes or lat-long.
*/
$dbhandle = @database_connect_command( 'somehost', 'someuser', 'somepassword', 'somedatabase' );
if( !$dbhandle ) {
print '<p>ERROR: could not open database</p>';
}
database_query_command( 'CREATE TABLE `'.addslashes($_POST['grdname']).'` ( recordindex int(10), latindex INT(5), longindex INT(5), latitude FLOAT, longitude FLOAT, shift FLOAT, PRIMARY KEY( recordindex ) ) CHARSET=utf8', $dbhandle );
if( $error = database_error_command() ) {
print '<p>Error: '.htmlspecialchars($error).'</p>';
fclose($filehandle);
exit;
}
}
} else {
set_time_limit(60);
}
print '<p id="showloading">Processing. Please wait...</p>';
ob_flush();
flush();
$filename = $override_file_name ? $override_file_name : $_FILES['grdfile']['tmp_name'];
$starttime = microtime(true);
$filehandle = @fopen( $filename, 'r' );
$haderror = false;
$alldata = array();
$headerread = false;
$linenumber = 0;
$datacount = 0;
$highest = -INF;
$lowest = INF;
$filebytes = 0;
$latmin = $latmax = $longmin = $longmax = $latspacing = $longspacing = $latstepdp = $longstepdp = $latcount = $longcount = null;
if( $filehandle ) {
$numreg = "(-?\d+\.?|-?\d*\.\d+)";
while( ( $line = fgets($filehandle) ) !== false ) {
$filebytes += strlen($line); //measured rather than using the upload size, to allow local files to be used instead
$linenumber++;
//if there is such a thing as comments in these files, they can be removed here
if( preg_replace( "/\s+/", '', $line ) ) {
if( !$headerread && preg_match( "/^\s*$numreg\s+$numreg\s+$numreg\s+$numreg\s+$numreg\s+$numreg\s*$/", $line, $lineparts ) ) {
//read the values out of a header - the first line that is not blank (ignoring comments, if support has been added for comments)
$latmin = $lineparts[1];
$latmax = $lineparts[2];
$longmin = $lineparts[3];
$longmax = $lineparts[4];
$latspacingspec = $lineparts[5];
$longspacingspec = $lineparts[6];
$latstepdp = max( strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[1] * 1 ) ), strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[5] * 1 ) ) );
$longstepdp = max( strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[3] * 1 ) ), strlen( preg_replace( "/^-?\d*\.?/", '', $lineparts[6] * 1 ) ) );
$latdiff = $latmax - $latmin;
$longdiff = $longmax - $longmin;
if( ( $latspacingspec * 1 ) <= 0 || ( $longspacingspec * 1 ) <= 0 ) {
$haderror = true;
print '<p>ERROR: header on line $linenumber does not supply valid (non-zero) values for latitude or longitude spacing, got; ' . htmlspecialchars($latspacing) . ' and ' . htmlspecialchars($longspacing) . '</p>';
break;
}
//OSI data has a rounding precision problem that causes it to miss its end point; 0.013333 is not the same as 5 / 375
//therefore this code tries to work out the actual spacing, by seeing how many data points can fit with approximately that spacing
//and then adjusting the spacing accordingly
if( $latdiff ) {
$latcount = round( $latdiff / $latspacingspec );
$latspacing = $latdiff / $latcount;
//fencepost problem
$latcount++;
} else {
$latcount = 1;
}
if( $longdiff ) {
$longcount = round( $longdiff / $longspacingspec );
$longspacing = $longdiff / $longcount;
//fencepost problem
$longcount++;
} else {
$longcount = 1;
}
$headerread = true;
} elseif( $headerread ) {
//every line after the header should be a series of numbers
//usually they are grouped into blocks of several lines containing 8 data point values, with blank lines separating
//groups which add up to a single line of latitude
//the exact number does not matter, as long as the file has the right number of data points overall
$datapoints = preg_split( "/\s+/", trim($line) );
foreach( $datapoints as $onenumber ) {
if( !preg_match( "/^(-?\d\.?|-?\d*\.\d+)$/", $onenumber ) ) {
$haderror = true;
print "<p>ERROR: non-numeric data found where numeric data was expected on line $linenumber; " . htmlspecialchars($onenumber) . '</p>';
break 2;
}
$latindex = floor( $datacount / $longcount );
$longindex = $datacount % $longcount;
$latitude = $latmax - $latindex * $latspacing;
$longitude = $longmin + $longindex * $longspacing;
if( !( $longindex ) ) {
$alldata[$latindex] = array();
}
$onenumber *= 1;
//storing the data in an array uses a load of memory, but this is what allows it to only process the data when it has parsed correctly
//it would be possible to send it to the browser for rendering at this point, but then a parsing error would result in a script error
if( $_POST['dataaction'] != 'addlowmem' ) {
$alldata[$latindex][$longindex] = ( $_POST['dataaction'] == 'render' || $_POST['dataaction'] == 'renderaddlowmem' ) ? $onenumber : array( $latitude, $longitude, $onenumber );
}
if( $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'addlowmem' ) {
put_data_in_database( $longcount * $latindex + $longindex, $latindex, $longindex, $latitude, $longitude, $latstepdp, $longstepdp, $onenumber, $_POST['grdname'] );
}
if( $onenumber > $highest ) {
$highest = $onenumber;
}
if( $onenumber < $lowest ) {
$lowest = $onenumber;
}
$datacount++;
}
} else {
$haderror = true;
print "<p>ERROR: file did not start with a header on line $linenumber, expected 6 numbers separated by spaces, got; " . htmlspecialchars($line) . '</p>';
break;
}
}
}
fclose($filehandle);
if( !$haderror ) {
if( !$headerread ) {
$haderror = true;
print '<p>ERROR: got to the end of the file, and did not discover a header.</p>';
} elseif( $datacount != $latcount * $longcount ) {
$haderror = true;
print '<p>ERROR: expected ' . ( $latcount * $longcount ) . " data points but got $datacount.</p>";
}
}
} else {
$haderror = true;
print '<p>ERROR: could not open file ' . htmlspecialchars($filename) . '</p>';
}
$endtime = microtime(true);
if( !$haderror ) {
if( $_POST['dataaction'] == 'renderadd' || $_POST['dataaction'] == 'add' ) {
/*
THIS IS WHERE YOU CONNECT TO THE DATABASE
$dbhandle = your database connection handle
*/
$dbhandle = @database_connect_command( 'somehost', 'someuser', 'somepassword', 'somedatabase' );
if( !$dbhandle ) {
print '<p>ERROR: could not open database</p>';
fclose($filehandle);
exit;
}
database_query_command( 'CREATE TABLE `'.addslashes($_POST['grdname']).'` ( recordindex int(10), latindex INT(5), longindex INT(5), latitude FLOAT, longitude FLOAT, shift FLOAT, PRIMARY KEY( recordindex ) ) CHARSET=utf8', $dbhandle );
if( $error = database_error_command() ) {
print '<p>Error: '.htmlspecialchars($error).'</p>';
fclose($filehandle);
exit;
}
for( $latindex = 0; $latindex < $latcount; $latindex++ ) {
for( $longindex = 0; $longindex < $longcount; $longindex++ ) {
put_data_in_database( $longcount * $latindex + $longindex, $latindex, $longindex, $alldata[$latindex][$longindex][0], $alldata[$latindex][$longindex][1], $latstepdp, $longstepdp, $alldata[$latindex][$longindex][2], $_POST['grdname'] );
}
}
//process any remaining batches
put_data_in_database( null, null, null, null, null, null, null, null, null, true );
} elseif( $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'addlowmem' ) {
put_data_in_database( null, null, null, null, null, null, null, null, null, true );
}
if( $_POST['dataaction'] == 'render' || $_POST['dataaction'] == 'renderaddlowmem' || $_POST['dataaction'] == 'renderadd' ) {
//prepare to render the data using HTML canvas
$datarange = $highest - $lowest;
if( !$datarange ) {
//avoid dividing by 0
$datarange = 0.000001;
}
print '<script type="text/javascript">document.getElementById("showloading").textContent = "Loading results. Please wait...";</script>';
ob_flush();
flush();
//build data[latindex][data,data,...,data] arrays
print '<p><canvas width="' . $longcount . '" height="' . $latcount . '"></canvas></p><script type="text/javascript">
var datarange = ' . $datarange . ', canvasdata = [' . "\n";
foreach( $alldata as $onerow ) {
print '[';
foreach( $onerow as $onecell ) {
print ( $highest - ( ( $_POST['dataaction'] == 'renderadd' ) ? $onecell[2] : $onecell ) ) . ',';
}
print "],\n";
}
print "];
function paintCanvas( contour ) {
var canvas = document.getElementsByTagName('canvas')[0];
var context = canvas.getContext('2d'), lightness;
context.clearRect( 0, 0, canvas.width, canvas.height );
for( var y = 0; y < canvasdata.length; y++ ) {
for( var x = 0; x < canvasdata[y].length; x++ ) {
if( !contour ) {
lightness = '50';
} else if( contour == 1 ) {
lightness = ( Math.floor( canvasdata[y][x] % 10 ) ? '5' : '4' ) * 10;
} else if( contour == 2 ) {
lightness = ( Math.floor( ( canvasdata[y][x] / 5 ) % 2 ) + 4 ) * 10;
} else if( contour == 3 ) {
lightness = Math.floor( canvasdata[y][x] % 10 ) * 10;
} else if( contour == 4 ) {
lightness = ( Math.floor( ( 10 * canvasdata[y][x] ) % 10 ) ? '5' : '4' ) * 10;
} else if( contour == 5 ) {
lightness = ( Math.floor( ( 2 * canvasdata[y][x] ) % 2 ) + 4 ) * 10;
} else if( contour == 6 ) {
lightness = Math.floor( ( 10 * canvasdata[y][x] ) % 10 ) * 10;
} else if( contour == 7 ) {
lightness = ( Math.floor( ( 100 * canvasdata[y][x] ) % 10 ) ? '5' : '4' ) * 10;
} else if( contour == 8 ) {
lightness = ( Math.floor( ( 20 * canvasdata[y][x] ) % 2 ) + 4 ) * 10;
} else if( contour == 9 ) {
lightness = Math.floor( ( 100 * canvasdata[y][x] ) % 10 ) * 10;
}
if( contour == 10 ) {
context.fillStyle = 'hsl( ' + ( 300 * canvasdata[y][x] / datarange ) + ', 100%, ' + ( 100 - canvasdata[y][x] * 100 / datarange ) + '% )';
} else if( contour == 11 ) {
context.fillStyle = 'hsl( 0, 0%, ' + ( 100 - canvasdata[y][x] * 100 / datarange ) + '% )';
} else {
context.fillStyle = 'hsl( ' + ( 300 * canvasdata[y][x] / datarange ) + ', 100%, ' + lightness + '% )';
}
context.fillRect( x, y, 1, 1 )
}
}
}
</script>
<p><label for='contouring'>Contouring: <select id='contouring' onchange='paintCanvas(this.selectedIndex);'>
<option>None</option>
<option>Light 10 m</option>
<option>Moderate 10 m</option>
<option>Heavy 10 m</option>
<option>Light 1 m</option>
<option>Moderate 1 m</option>
<option>Heavy 1 m</option>
<option>Light 0.1 m</option>
<option>Moderate 0.1 m</option>
<option>Heavy 0.1 m</option>
<option>Gradient</option>
<option>Greyscale</option>
</select></label></p>";
}
print "<dl>
<dt>Latitude min</dt>
<dd>$latmin</dd>
<dt>Latitude max</dt>
<dd>$latmax</dd>
<dt>Longitude min</dt>
<dd>$longmin</dd>
<dt>Longitude max</dt>
<dd>$longmax</dd>
<dt>Latitude spacing</dt>
<dd>$latspacingspec</dd>
<dt>Longitude spacing</dt>
<dd>$longspacingspec</dd>
<dt>Latitude spacing decimal places</dt>
<dd>$latstepdp</dd>
<dt>Longitude spacing decimal places</dt>
<dd>$longstepdp</dd>
</dl>
<script type='text/javascript'>
document.getElementById('showloading').textContent = 'Rendering. Please wait...';
</script>";
$imagetime = microtime(true);
print "<p id='stats'>Highest point discovered $highest, lowest point discovered $lowest.
File contained $datacount points in $latcount rows of $longcount columns.
Processing $filebytes bytes took " . ( $endtime - $starttime ) . ' seconds and ' . ( memory_get_usage() / ( 1024 * 1024 ) ) . 'MB of memory.
HTML output took ' . ( $imagetime - $endtime ) . ' seconds.</p>';
}
} elseif( isset($_FILES['grdfile']) ) {
//handle file upload errors
switch($_FILES['grdfile']['error']) {
case UPLOAD_ERR_INI_SIZE:
print '<p>ERROR: Uploaded file was too large. Max size ' . htmlspecialchars($maxsizereadable) . '.</p>';
break;
case UPLOAD_ERR_PARTIAL:
print '<p>ERROR: Uploaded file was partially incomplete; upload was probably interrupted.</p>';
break;
case UPLOAD_ERR_NO_FILE:
print '<p>ERROR: No file was uploaded.</p>';
break;
case UPLOAD_ERR_NO_TMP_DIR:
print '<p>ERROR: Server configuration fault; no upload temporary directory was specified.</p>';
break;
case UPLOAD_ERR_CANT_WRITE:
print '<p>ERROR: Server fault; could not write to disk - disk full?</p>';
break;
case UPLOAD_ERR_EXTENSION:
print '<p>ERROR: Server configuration fault; a PHP extension halted the upload.</p>';
break;
default:
print '<p>ERROR: Unknown error type.</p>';
}
}
$maxsize = strtolower( $maxsizereadable );
$chars = preg_replace( "/[^a-z]/i", '', $maxsize );
$maxsize = intval($maxsize);
switch( $chars ) {
case 't':
$maxsize *= 1024;
case 'g':
$maxsize *= 1024;
case 'm':
$maxsize *= 1024;
case 'k':
$maxsize *= 1024;
default:
$maxsize *= 1;
}
?>
<form method="post" action="parsegrd.php" enctype="multipart/form-data">
<p><label for="grdfile">Select grd file (max <?php print htmlspecialchars($maxsizereadable); ?>):<br><input type="file" name="grdfile" id="grdfile" onchange="if( this.files[0].size > <?php print $maxsize; ?> ) { alert('File is too large. Max size is <?php print htmlspecialchars($maxsizereadable); ?>.'); this.value = ''; }"></label></p>
<p><label for="grdname">Identification name for the geoid set in the database:<br><input type="text" name="grdname" id="grdname"></label></p>
<p><label for="dataaction">What would you like to do with the parsed data (note, rendering sends it back to you as JavaScript data, large files will hurt):<br><select name="dataaction" id="dataaction">
<option value="render"<?php if( !isset($_POST['dataaction']) || $_POST['dataaction'] == "render" ) { print ' selected'; } ?>>Render if parsing is successful (medium memory usage)</option>
<option value="renderaddlowmem"<?php if( isset($_POST['dataaction']) && $_POST['dataaction'] == "renderaddlowmem" ) { print ' selected'; } ?>>Add to database during parsing, and render if parsing is successful (medium memory usage)</option>
<option value="renderadd"<?php if( isset($_POST['dataaction']) && $_POST['dataaction'] == "renderadd" ) { print ' selected'; } ?>>Add to database if parsing is successful, and render if parsing is successful (high memory usage)</option>
<option value="addlowmem"<?php if( isset($_POST['dataaction']) && $_POST['dataaction'] == "addlowmem" ) { print ' selected'; } ?>>Add to database during parsing (low memory usage)</option>
<option value="add"<?php if( isset($_POST['dataaction']) && $_POST['dataaction'] == "add" ) { print ' selected'; } ?>>Add to database if parsing is successful (high memory usage)</option>
</select></label><br><strong>WARNING:</strong> if you add to the database during parsing, and parsing fails part way through, the database will contain incomplete data.</p>
<p><input type="submit" value="Upload"></label></p>
</form>
</body>
</html>