#!/usr/bin/perl # File: filtergulp # This program filters the cooked data out of a No Average DAS file # and generates a fixed field, comma seperated ascii text file. # The first line is a comma seperated list of headers for each field. # Adapted from filterdas on 01 May 2005 by Toby Martin # Modified: use Time::Local; # # Set constants. # $ERROR_CODE = -999999; $pi = atan2 (1, 1) * 4; $degtorad = $pi / 180; $radtodeg = 180 / $pi; $hours_per_day = 24; $minutes_per_hour = 60; $seconds_per_minute = 60; $seconds_per_hour = $minutes_per_hour * $seconds_per_minute; $seconds_per_day = $hours_per_day * $seconds_per_hour; $DEFAULT_LISTNAME = "filter.list"; # Need this so it displays correctly # in the usage. # # Set default values, these can be overridden on the command line. # See CK_arguments for the overrides. # $FILTER_LIST_FILENAME = $DEFAULT_LISTNAME; $PROCESS_YEAR = (gmtime (time))[5] + 1900; $DELIMITER =","; $file_size = "daily"; $debug = 0; # False CK_arguments (); # Process the command line arguments. Read_output_list (); # Load the configuration of the output list. while (<>) { chomp (); my $input_line = $_; my $sensor; if (/\$PSTA/) { CK_print(); } # Process the line if ((/PFREQ/)||(/PUNIT/)||(/PVOLT/)) { ($code, $sensor, $remainder) = split (/,/, $input_line, 3); $data_record {$sensor} = $sensor . "," . $remainder; } elsif ($input_line) # not blank { ($sensor) = split (/,/, $input_line); $data_record {$sensor} = $input_line; } if (/\$PEND/) { CK_print(); } } # while port ################################################################ ##################### end of main routine ##################### ##################### start of subroutines ##################### ################################################################ # # Derive useful values # sub Derive_values { { # truetime my $truetime = (Parse_csv ($data_record{"\$ZAZDA"}))[1]; my ($day, $hour, $minute, $second) = split (/:/, $truetime); my $decimal_day = $day + ((($hour * $seconds_per_hour) + ($minute * $seconds_per_minute) + $second) / $seconds_per_day); $decimal_day = sprintf ("%1.5f", $decimal_day); my $start_of_year = (timegm (0, 0, 0, 1, 0, $PROCESS_YEAR)) - $seconds_per_day; my $elapsed = $day * $seconds_per_day + $hour * $seconds_per_hour + $minute * $seconds_per_minute + $second; my $cur_time = $start_of_year + $elapsed; my ($ttt_sec, $ttt_minute, $ttt_hour, $ttt_day, $ttt_month, $ttt_year) = gmtime ($cur_time); my $ymd_hm = sprintf("%4d-%02d-%02d %02d:%02d", $ttt_year + 1900, $ttt_month + 1, $ttt_day, $ttt_hour, $ttt_minute); my $ymd_hms = sprintf("%4d-%02d-%02d %02d:%02d:%02d", $ttt_year + 1900, $ttt_month + 1, $ttt_day, $ttt_hour, $ttt_minute, $ttt_sec); $data_record {"truetime"} = "truetime," . $decimal_day . "," .$ymd_hm . "," .$ymd_hms; } # truetime { # P-Code my $pcode = $data_record{"\$GPGGA"}; my ($code, $pc_time, $pc_lat, $pc_ns, $pc_long, $pc_ew, $pc_quality, @pc_remainder) = Parse_csv ($pcode); # Seperate hours and minutes out of time. my $pc_hh = substr ($pc_time, 0, 2); my $pc_mm = substr ($pc_time, 2, 2); my $pc_ss = substr ($pc_time, 4, 2); # Seperate latitude degrees from decimal minutes. my ($pc_lat_deg, $pc_lat_min, $pc_lat_frac) = split (/(\d\d\.)/, $pc_lat); $pc_lat_min = $pc_lat_min . $pc_lat_frac; my $pc_lat_decimal_degree = $pc_lat_deg + ($pc_lat_min / 60); if ($pc_ns =~ /S/) { # Southern Hemisphere $pc_lat_decimal_degree = $pc_lat_decimal_degree * -1; } # Southern Hemisphere # Seperate longitude degrees from decimal minutes. my ($pc_long_deg, $pc_long_min, $pc_long_frac) = split (/(\d\d\.)/, $pc_long); $pc_long_min = $pc_long_min . $pc_long_frac; my $pc_long_decimal_degree = $pc_long_deg + ($pc_long_min / 60); if ($pc_ew =~ /W/) { # Western Hemisphere $pc_long_decimal_degree = $pc_long_decimal_degree * -1; } # Western Hemisphere my $pcode_string = sprintf ("pcode,%02d:%02d:%02d,%.6f,%.6f", $pc_hh, $pc_mm, $pc_ss, $pc_lat_decimal_degree, $pc_long_decimal_degree); $data_record {"pcode"} = $pcode_string; } # P-Code } #sub Derive_values # # Calculate True Winds # sub Calc_truewind { my $wind_speed = $_[0]; my $wind_heading = $_[1]; my $ship_speed = $_[2]; my $ship_heading = $_[3]; my $true_wind_heading, $true_wind_speed; # Calculate true wind speed and heading from starboard sensors. if (!( Is_numeric ($wind_speed) && Is_numeric ($wind_heading) && Is_numeric ($ship_speed) && Is_numeric ($ship_heading) )) { # One of the components is bad $true_wind_heading = $ERROR_CODE; $true_wind_speed = $ERROR_CODE; } # One of the components is bad else { # Components are all numeric my $relative_wind_speed; my $relative_wind_heading; my $relative_wind_heading_radians; my $wind_speed_transverse; my $wind_speed_longitudinal; $relative_wind_speed = $wind_speed * 0.02; $relative_wind_heading = $wind_heading * 0.072; $relative_wind_heading_radians = $relative_wind_heading * $degtorad; $wind_speed_transverse = -$relative_wind_speed * sin($relative_wind_heading_radians); $wind_speed_longitudinal = -$relative_wind_speed * cos($relative_wind_heading_radians) + $ship_speed; # atan2 (x, y) returns arctangent in range -pi to +pi. $true_wind_heading = 270 - (atan2 ($wind_speed_longitudinal, $wind_speed_transverse) * $radtodeg) + $ship_heading; $true_wind_heading = Mod_circle ($true_wind_heading); $true_wind_speed = sqrt ($wind_speed_transverse * $wind_speed_transverse + $wind_speed_longitudinal * $wind_speed_longitudinal); # Format to 4 place past the decimal $true_wind_heading = sprintf ("%1.4f", $true_wind_heading); $true_wind_speed = sprintf ("%1.4f", $true_wind_speed); } # Components are all numeric return ($true_wind_heading, $true_wind_speed); } # sub Calc_truewind # # Truncate degrees to 0-360 # sub Mod_circle { my $value = $_[0]; while ($value < 0) { $value += 360; } while ($value > 360) { $value -= 360; } return ($value); } # sub Mod_circle sub Mod_half_circle { my $value = $_[0]; while ($value < -180) { $value += 360; } while ($value > 180) { $value -= 360; } return ($value); } # sub Mod_half_circle # # Trim leading and trailing whitespace # sub Trim { my @out = @_; # Declare a local array, initialize to param array. for (@out) { # Replace one or more whitespace at begining of string with nothing. s/^\s+//; # Replace one or more whitespace at end of string with nothing. s/\s+$//; } # for loop # Return list or scalar, depending on calling context. return wantarray ? @out : $out[0]; } # sub Trim # # Print header # sub CK_print_header { Debug_print (5, "Enter: CK_print_header"); $number_vars = $#order; for ($loop_count = 0; $loop_count < $number_vars; $loop_count++) { CK_print_item ($description{$order[$loop_count]}); } # for loop_count CK_print_item ($description{$order[$number_vars]}, "last"); Debug_print (5, "Leave: CK_Print_header"); } # CK_print_header # # Open the cooked output file # sub CK_open { my ($cooked_name, $empty_file); # Determine filename. if ($OUTPUT_FILENAME) { $cooked_name = $OUTPUT_FILENAME; } else { my $yearday; my $das_filename = (Parse_csv ($data_record{"\$PSTA"}))[3]; $cooked_name = Trim ($das_filename); $cooked_name =~ /(\d\d\d)(.txt)/; $yearday = $1; if ($file_size eq "daily") {$cooked_name = $yearday . ".ck";} else # hourly { my $date = (Parse_csv ($data_record{"\$ZAZDA"}))[1]; my $hour = (split (/:/, $date))[1]; $cooked_name = $yearday . "-". $hour . ".ck"; } } if (-e $cooked_name) { # if file exists Debug_print (6, "File $cooked_name already exists."); $empty_file = 0; # False } # file exists else { Debug_print (6, "File $cooked_name does not exist."); $empty_file = 1; # True } # file does not exist # Open file in append mode. open (COOKOUT, ">> $cooked_name") or die "Couldn't open $cooked_name for appending: $!\n"; return ($empty_file); Debug_print (5, "Leave CK_open, file=$cooked_name"); } # sub CK_open # # Print out a filtered record # sub CK_print { if ($data_record{"\$PSTA"}) # If a full record { my $empty_file = CK_open (); # Open cooked output file if ($empty_file) { CK_print_header (); # Print a header in the cooked file. } Derive_values (); { # Print block my ($sensor, $index, $value); $number_vars = $#order; for ($loop_count = 0; $loop_count < $number_vars; $loop_count++) { ($sensor, $index) = split (/:/, $order[$loop_count]); $value = (Parse_csv ($data_record{$sensor}))[$index]; # print "sensor=$sensor index=$index value=$value\n"; CK_print_item ($value); } # for loop_count ($sensor, $index) = split (/:/, $order[$number_vars]); $value = (Parse_csv ($data_record{$sensor}))[$index]; # print "sensor=$sensor index=$index value=$value ***\n"; CK_print_item ($value, "last"); } # Print block close (COOKOUT); } # if PSTA exists # clear hashes %data_record = (); Debug_print (5, "Leave: CK_print"); } # sub CK_print # # Print an individual field in a comma seperated format. # If field is empty, print and error code. # sub CK_print_item { my $item_value = Trim ($_[0]); Debug_print (9, "CK_print_item: @_"); if (($item_value !~ /.+/) # Empty string, no characters. || ($item_value =~ /^-$/)) # Just a singlle dash. { # if empty print COOKOUT "$ERROR_CODE"; } # if empty else { # not empty print COOKOUT "$item_value"; } # not empty if ($_[1] eq "last") { print COOKOUT "\n"; } else { print COOKOUT "$DELIMITER"; } } # sub CK_print_item # # Parse a string into comma seperated values. # Taken from the "Perl Cookbook", Christian & Torkington, example 1.15 # sub Parse_csv { use Text::ParseWords; return quotewords (",", 0, $_[0]); } # # Print usage information # sub Usage { print "Usage: filterdas [OPTIONS] [DAS_FILE]...\n"; print "Filter selected fields out of DAS \".txt\" file(s).\n\n"; print " -hourly one output file per day, default=daily\n"; print " -debug level don't use this!\n"; print " -delimit=c change output delimiter to c\n"; print " -help display this help and exit\n"; print " -l list_file specify filter_list file, default=$DEFAULT_LISTNAME\n"; print " -o out_file specify output file, default=daily from header\n"; print " -version output version information and exit\n"; print " -y YYYY specify the year the data was collected, default=current\n"; print "\nIf no DAS_FILE, read standard input.\n\n"; print "Report bugs to suds\@oce.orst.edu\n"; } # sub Usage # # Process the command line arguments. # sub CK_arguments { # $argc = $#ARGV + 1; # $argc = $#ARGV; # print "argc = $argc\n"; if ($#ARGV == -1) { # invalid option Usage (); die "\nError: No arguments\n"; } while ($_ = $ARGV[0], /^-/) # while arguments start with a minus. { shift @ARGV; # shift first argument into workspace last if /^--$/; # exit loop if STDIN if (/^-debug/) { # if debug $debug = shift @ARGV; # next argument is debug level } elsif (/^-hourly/) { # if hourly $file_size = "hourly"; # set filesize to hourly } elsif (/(^-delimit=)(.+)/) { $DELIMITER = $2; } elsif (/^-help/) { Usage (); die "\n"; } # This is a good exaple of how to use regular expressions to seperate # two tokens with nothing between them. # elsif (/(^-o)(.+)/) # look for -o # { # Set ouptut list file # $FILTER_LIST_FILENAME = $2; # print "Output list: $FILTER_LIST_FILENAME\n"; # } elsif (/^-l/) { # follwing argument is FILTER_LIST_FILENAME $FILTER_LIST_FILENAME = shift @ARGV; # die "Output list filename is $FILTER_LIST_FILENAME\n"; } elsif (/^-o/) { $OUTPUT_FILENAME = shift @ARGV; # die "Output list filename is $FILTER_LIST_FILENAME\n"; } elsif (/^-version/) # output version information and exit { die "filterdas, 24 Jan 2000\n"; } elsif (/^-y/) # Set the $PROCESS_YEAR { $PROCESS_YEAR = shift @ARGV; } else { # invalid option Usage (); die "\nError: unknown option \"$_\"\n"; } } # while Debug_print (3, "filesize=$file_size"); Debug_print (3, "debug=$debug"); } # sub CK_arguments # # Print debugging messages. # levels: # 1: most important, almost never used # 3: important, main stages of the program # 5: typical, enter/leave main functions # 7: variables in main functions # 9: trivial, inside loops or functions that are often used # sub Debug_print { my $level = shift (@_); if ($level <= $debug) { print "@_\n"; } } # sub Debug_print # # Check to see if the token is numeric. # From: Perl Cookbook, Christiansen & Torkington, problem 2.1, pages 44-45. # sub Is_numeric { my $value = $_[0]; $value =~ s/^\s+//; # Strip leading whitspace. $value =~ s/\s+$//; # Strip trailing whitespace. return ( $value =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/ ); } # sub Is_numeric # # Read the configuration of the output list. # sub Read_output_list { %description = (); # Clear the hash; open (USER_CONFIG_FILE, "< $FILTER_LIST_FILENAME") or die "Couldn't open $FILTER_LIST_FILENAME for reading: $!\n"; while () { chomp (); my ($sensor, $descrip) = split (); push (@order, $sensor); $description {$sensor} = $descrip; } # while USER_CONFIG_FILE } # Read_output_list