[OpenMadrigal-developers] Re: Files -- madrigal loading
William Rideout
brideout at haystack.mit.edu
Wed Oct 15 15:43:57 EDT 2003
Angela Li wrote:
> Hi Bill and Tony,
>
> I excluded madwish on the Makefile.gnu before I built with the following instructions. Then I get the following message:
>
Angela,
Again this is a problem with ffw. I have two solutions for you
(hopefully this doubles the chances of solving the problem). You can
use either one (or both):
1. Use the attached modified script Load_Files_to_MADRIGAL2.exp. It
uses isprint instead of ffw.
--or--
2. Fix ffw as follows:
a) copy attached file ffw_print.tcl to $MADROOT/source/madtcl/bin
b) cd $MADROOT
c) tclsh configureScripts
This should fix ffw, and allow your original script to run.
Technical discussion:
The present implementation of ffw has a very few hard-coded derived
parameters it uses. These methods are marked as deprecated in the C
API, since they use hard-coded parameter codes, which John and I since
changed. I kludged code in the tcl script to get around this problem
for the month and day parameters that Tony uses here, but now I'm
convinced I'll change ffw in the next release to be simply an
alternative version of isprint (and remove the deprecated methods in the
C API).
Bill
--
Bill Rideout
MIT Haystack Observatory
Email: brideout at haystack.mit.edu
Phone: 781 981-5624
-------------- next part --------------
#!/bin/sh
# The madtclsh path is longer than 32 characters. So, we take advantage
# of the fact that a backslash continues a comment line in tcl \
exec /opt/madrigal-2.2.1/bin/madtclsh "$0" ${1+"$@"}
# Script to automate loading into MADRIGAL and updating necessary files
# ***** set MADROOT before running! ******
lappend auto_path /opt/madrigal-2.2.1/madtcllib
package require cgi
package require mtl
# Stage directory should contain the following set of files for each experiment:
# acport*sigma analysed data in NCAR format
# *.txt single line experiment description (optional, but recommended)
# YYMMDD*.ps matching PostScript plots (optional, but recommended)
#
# handle Sondrestromfjord data only!
set siteName "son"
set siteCode 80
madExperiment experiment
set madroot [experiment cget -madroot]
set madserver [experiment cget -madserver]
set madservercgi [experiment cget -madservercgi]
set madserverroot [experiment cget -madserverroot]
set htmlstyle [experiment cget -htmlstyle]
set StageDirectory "$madroot/experiments/stage"
set MADRIGALDirectory "$madroot/experiments"
set WWWMADRIGALroot "http://$madserver/$madserverroot/experiments"
# Read in list of private KINDATS, if it exists, otherwise create an empty list
if { ![ catch { exec /bin/cat $madroot/privateKINDATs.txt } privateKINDATsFile ] } {
set privateKINDATEntries [ split $privateKINDATsFile "\n" ]
foreach privateKINDATEntry $privateKINDATEntries {
set privateKINDATs [ lappend privateKINDATs [ string trim $privateKINDATEntry ] ]
}
} else {
set privateKINDATs [ list ]
}
# This script will automatically inform those in the email list (if present)
if { ![ catch { exec /bin/cat $madroot/updateMailingList.txt } returnMessage ] } {
set emailListEntries [ split $returnMessage "\n" ]
foreach emailListEntry $emailListEntries {
set emailList [ lappend emailList [ string trim $emailListEntry ] ]
}
} else {
set emailList {}
}
puts $emailList
set mailSubject "Notification of data added to SRI MADRIGAL"
# Procedure converts a date such as 990823 to the MADRIGAL directory name form 23aug99
#
proc YYmmDDToMADRIGALDate { YYmmDD } {
set monthNames(1) "jan"; set monthNames(2) "feb"; set monthNames(3) "mar"
set monthNames(4) "apr"; set monthNames(5) "may"; set monthNames(6) "jun"
set monthNames(7) "jul"; set monthNames(8) "aug"; set monthNames(9) "sep"
set monthNames(10) "oct"; set monthNames(11) "nov"; set monthNames(12) "dec"
set monthNumber [ string range $YYmmDD 2 3 ]
if { [ string range $monthNumber 0 0 ] == "0" } {
set monthNumber [ string range $monthNumber 1 1 ]
}
return "[ string range $YYmmDD 4 5 ]$monthNames($monthNumber)[ string range $YYmmDD 0 1 ]"
}
#
# Main program starts here
#
puts "\n================================================================"
puts "\nDATA LOAD to the SRI MADRIGAL site...."
puts "\nThis script will load data from a specified directory to"
puts "http://$madserver/$madserverroot and copy any *sigma analysed files,"
puts "PostScript, jpeg, and gif images to the appropriate directories.\n"
if { [ info exists env(MADROOT) ] } {
puts "MADROOT: $env(MADROOT)"
} else {
puts "\a\a\a\aPlease set MADROOT (export MADROOT=$madroot) and restart\n"
exit
}
# Get a list of the experiments (ie all the *sigma files) to be processed
set datafileList {}; set datafilesFound 0
set GIFfileList {}; set GIFfilesFound 0
set PSfileList {}; set PSfilesFound 0
puts "\nIs the data to be loaded in a single directory?"
gets stdin userInput
if { [ string range $userInput 0 0 ] != "Y" && [ string range $userInput 0 0 ] != "y" } {
puts "Please enter the full name of the file containing the data list"
gets stdin dataList
if { ![ catch { exec /bin/cat $dataList | grep sigma | egrep "acport|velbinport|cwinds|epec|pecport|ewinds|fwinds|veest|peest" } datafileList ] } {
set datafilesFound 1
set StageDirectory ""
}
if { ![ catch { exec /bin/cat $dataList | egrep ".jpeg\$|.jpg\|.gif\$" } GIFfileList ] } {
set GIFfilesFound 1
set StageDirectory ""
}
if { ![ catch { exec /bin/cat $dataList | grep ".ps\$" } PSfileList ] } {
set PSfilesFound 1
set StageDirectory ""
}
} else {
puts "Please enter the full path name of the directory containing the data"
puts "(hit return to use $StageDirectory)"
gets stdin userInput
if { [ string length $userInput ] != 0 } {
set StageDirectory $userInput
}
if { ![ catch { exec /bin/ls -1 $StageDirectory | grep sigma | egrep "acport|velbinport|cwinds|epec|pecport|ewinds|fwinds|veest|peest" } datafileList ] } {
set datafilesFound 1
set i 0
foreach fileEntry $datafileList {
lreplace $datafileList $i $i $StageDirectory$fileEntry
incr i
}
}
if { ![ catch { exec /bin/ls -1 $StageDirectory | egrep ".jpeg\$|.jpg\|.gif\$" } GIFfileList ] } {
set GIFfilesFound 1
set i 0
foreach fileEntry $GIFfileList {
lreplace $GIFfileList $i $i $StageDirectory$fileEntry
incr i
}
}
if { ![ catch { exec /bin/ls -1 $StageDirectory | ".grep ".ps\$" } PSfileList ] } {
set PSfilesFound 1
set i 0
foreach fileEntry $PSfileList {
lreplace $PSfileList $i $i $StageDirectory$fileEntry
incr i
}
}
}
if { $datafilesFound } {
foreach fileEntry $datafileList {
puts "\nProcessing $StageDirectory$fileEntry"
set errorMessage 0
# convert the first record of the file and extract its Year Month and Day
set temporaryFile "/tmp/[pid]testOut"
if { ![ catch { exec $madroot/bin/mergeCedarFiles -i $StageDirectory/$fileEntry 1 1 -o $temporaryFile -t 1 } returnMessage ] } {
if { ![ catch { exec $madroot/bin/isprint header=f file=$temporaryFile -p year } returnMessage ] } {
set YYYY [ lindex $returnMessage 10 ]
if { [ string range $YYYY 0 0 ] == 1 || [ string range $YYYY 0 0 ] == 2 } {
if { ![ catch { exec $madroot/bin/isprint header=f file=$temporaryFile -p month } returnMessage ] } {
set MM [ lindex $returnMessage 10 ]
if { ![ catch { exec $madroot/bin/isprint header=f file=$temporaryFile -p day } returnMessage ] } {
set DD [ lindex $returnMessage 10 ]
# construct date in the form YYMMDD
set YY [ string range $YYYY 2 3 ]
if { [ string length $MM ] != 2 } { set MM "0$MM" }
if { [ string length $DD ] != 2 } { set DD "0$DD" }
set YYMMDD "$YY$MM$DD"
puts "Identified data from $YYMMDD"
} else { set errorMessage "ffw: $returnMessage" }
} else { set errorMessage "ffw: $returnMessage" }
} else { set errorMessage "Time machine warning: input appears to be for year $YYYY" }
} else { set errorMessage "ffw: $returnMessage" }
} else { set errorMessage "mergeCedarFiles: $returnMessage" }
catch { exec /bin/rm $temporaryFile } returnMessage
if { $errorMessage == 0 } {
# check if experiment already exists in MADRIGAL and get a title line if not
set experimentDirectory "$MADRIGALDirectory/$YYYY/$siteName/[ $YYMMDD ]"
if { [ catch { exec /bin/cat $experimentDirectory/expTab.txt } returnMessage ] } {
set elementsList [ split $returnMessage "," ]
if { [ lindex $elementsList 2 ] == "" } {
puts "Searching $madroot/MMcatalog.txt for titles...."
set expTitle ""
# Find all occurences of the date in the catalogue, extract and concatenate titles"
if { ![ catch { exec /bin/grep $YYMMDD $madroot/MMcatalog.txt } returnMessage ] } {
set catalogEntries [ split $returnMessage "\n" ]
foreach catalogEntry $catalogEntries {
set catalogFields [ split $catalogEntry "\t" ]
if { [ string first $YYMMDD [ lindex $catalogFields 0 ] ] != -1 } {
#puts "Found title: [ lindex $catalogFields 5 ]"
if { [ string length $expTitle ] == 0 } {
set expTitle [ lindex $catalogFields 5 ]
} else {
set expTitle "$expTitle + [ lindex $catalogFields 5 ]"
}
}
}
# Remove any commas, quotes and orphaned close bracket
regsub -all {,} $expTitle {} expTitle
regsub -all {"} $expTitle {} expTitle
if { [ string first "(" $expTitle ] == -1 } {
regsub -all {[)]} $expTitle {} expTitle
}
puts "Title: $expTitle"
} else {
# no titles found
puts "\n$YYYY-$MM-$DD: Enter short title (no commas!) for this day's data"
# for major dataload, set title empty
set expTitle ""
# gets stdin expTitle
while { [ string first "," $expTitle ] != -1 } {
puts "Please try again without commas!"
gets stdin expTitle
}
}
}
} else {
set expTitle "No need to set title when adding data to existing directory"
}
# Read input file and create a list of KINDAT entries
if { ![ catch { exec $madroot/bin/summarizeCedarFile $StageDirectory/$fileEntry } returnMessage ] } {
set recordSummaries [ split $returnMessage "\n" ]
set kindatList {}
foreach recordSummary $recordSummaries {
set newkindat [string trim [string range $recordSummary [string last " " $recordSummary] [string length $recordSummary]]]
if { [ lsearch $kindatList $newkindat ] == -1 && $newkindat != "kindat" } {
# kludge to handle empty records with illegal KINDAT codes, EOFs?
if { $newkindat != "16420" && $newkindat != "2055" && $newkindat != "2056" } {
puts " New kindat code found: $newkindat"
lappend kindatList $newkindat
}
}
}
} else { set errorMessage "summarizeCedarFile: $returnMessage" }
if { $errorMessage == 0 } {
# For each KINDAT: make a new file containing only that kindat and load it to madrigal
foreach kindat $kindatList {
puts " ___________________________________________________________________"
puts " Processing kindat = $kindat"
set fileSplit [file split $fileEntry]
set fileName [lindex $fileSplit [expr [llength $fileSplit]-1] ]
catch { exec /bin/rm /tmp/$fileName } returnMessage
if { ![ catch { exec $madroot/DataLoading/filterCedarFiles -k $kindat -i $StageDirectory/$fileEntry 1 99999 -o /tmp/$fileName -t 1 } returnMessage ] } {
if { ![ catch { exec $madroot/DataLoading/genExpFromFile /tmp/$fileName $siteCode $siteName $expTitle $YYMMDD} returnMessage ] } {
puts $returnMessage
set fileTabTxt "[string trim [string range $returnMessage [string last " " $returnMessage] [string length $returnMessage]]]/fileTab.txt"
} else { set errorMessage "genExpFromFile: $returnMessage" }
} else { set errorMessage "filterCedarFiles: $returnMessage" }
catch { exec /bin/rm -rf /tmp/$fileName } returnMessage
if { $errorMessage == 0 } {
if { ![ catch { exec /bin/cat $fileTabTxt } fileTabTxtEntries ] } {
set newEntry [ lindex $fileTabTxtEntries end ]
set fileTabParameters [ split $newEntry "," ]
# If kindat is in the list of private codes, update fileTab.txt entry
if { [ lsearch $privateKINDATs $kindat ] != -1 } {
set fileTabParameters [ lreplace $fileTabParameters 10 10 "1" ]
puts " Updated fileTab.txt to make data set private"
}
# Add original filename in location 09
set fileTabParameters [ lreplace $fileTabParameters 9 9 $fileName ]
set modifiedEntry [ join $fileTabParameters "," ]
# puts $modifiedEntry
set fileTabTxtEntries [ lreplace $fileTabTxtEntries end end $modifiedEntry ]
set fileTabTxtFile [ open $fileTabTxt "w" ]
foreach nextLine $fileTabTxtEntries {
puts $fileTabTxtFile $nextLine
}
close $fileTabTxtFile
} else {
puts $fileTabTxtEntries
}
# Copy original data file for this experiment to the
# appropriate experiment directory, gzip it, and set file permissions
if { [ catch { exec /bin/ls/ $experimentDirectory/$fileName.gz } returnMessage ] } {
# Only copy data file if a gzipped copy is not already present
# otherwise the gzip stage below will silently hang
catch { exec /bin/cp $fileEntry $experimentDirectory/$fileName } returnMessage
catch { exec /bin/gzip $experimentDirectory/$fileName } returnMessage
catch { exec /bin/chmod a+r $experimentDirectory/$fileName.gz } returnMessage
catch { exec /bin/chmod a-w $experimentDirectory/$fileName.gz } returnMessage
puts " Copied $fileEntry to experiment directory"
} else {
puts " Skipping copy of $fileName - gzip already exists"
}
# Update the INDEX file and inform the mailing list
set mailMessage "[ exec /bin/date ] added: $fileEntry at $YYYY/$siteName/[ YYmmDDToMADRIGALDate $YYMMDD ]"
catch { exec /bin/echo $mailMessage >> $madroot/INDEX_of_data_added_to_SRI_MADRIGAL } returnMessage
catch { exec /bin/rm /tmp/[ pid ]mailMessage } returnMessage
catch { exec /bin/echo "SRI MADRIGAL" > /tmp/[ pid ]mailMessage } returnMessage
catch { exec /bin/echo " " >> /tmp/[ pid ]mailMessage } returnMessage
catch { exec /bin/echo $mailMessage >> /tmp/[ pid ]mailMessage } returnMessage
catch { exec /bin/echo " " >> /tmp/[ pid ]mailMessage } returnMessage
catch { exec /bin/echo "Please direct queries to angela.li at sri.com" >> /tmp/[ pid ]mailMessage } returnMessage
foreach emailAddress $emailList {
set mailresult [ catch { exec /bin/mail $emailAddress -s $mailSubject < /tmp/[ pid ]mailMessage } returnMessage ]
}
catch { exec /bin/rm /tmp/[ pid ]mailMessage } returnMessage
puts "\a\a\a\a"
} else {
set errorMessage "Skipping data load, KINDAT = $kindat, error message was:\n$errorMessage"
catch { exec /bin/echo $fileEntry >> $madroot/LoadFailures } returnMessage
catch { exec /bin/echo $errorMessage >> $madroot/LoadFailures } returnMessage
puts $errorMessage
}
}
} else {
set errorMessage "Skipping data load, error message was:\n$errorMessage"
catch { exec /bin/echo $fileEntry >> $madroot/LoadFailures } returnMessage
catch { exec /bin/echo $errorMessage >> $madroot/LoadFailures } returnMessage
puts $errorMessage
}
} else {
set errorMessage "Skipping data load, error message was:\n$errorMessage"
catch { exec /bin/echo $fileEntry >> $madroot/LoadFailures } returnMessage
catch { exec /bin/echo $errorMessage >> $madroot/LoadFailures } returnMessage
puts $errorMessage
}
# try to keep the place tidy, even if failures occured
catch { exec /bin/rm /tmp/[ pid ]mailMessage } returnMessage
catch { exec /bin/rm /tmp/$fileName } returnMessage
catch { exec /bin/rm $temporaryFile } returnMessage
}
} else {
puts "\nNo data files found"
}
if { $GIFfilesFound } {
puts "\nCopying the following gif files into MADRIGAL..."
foreach fileEntry $GIFfileList {
set fileSplit [file split $fileEntry]
set fileName [lindex $fileSplit end ]
if { [ string match \[0-9\]\[0-9\]\[0-9\]\[0-9\]\[0-9\]\[0-9\] [string range $fileName 0 5] ] } {
set YYMMDD [ string range $fileName 0 5 ]
if { [ string range $YYMMDD 0 1 ] > 50 } {
set YYYY "19[ string range $YYMMDD 0 1 ]"
} else {
set YYYY "20[ string range $YYMMDD 0 1 ]"
}
# check if experiment exists in MADRIGAL and copy gif file there if it does
# (have to recheck since the plots may not belong to the present experiment)
set experimentDirectory "$MADRIGALDirectory/$YYYY/$siteName/[ YYmmDDToMADRIGALDate $YYMMDD ]"
if { ![ catch { exec /bin/ls $experimentDirectory } returnMessage ] } {
# Copy original data file for this experiment to the appropriate experiment directory
# and set file permissions
puts $fileEntry
catch { exec /bin/cp $fileEntry $experimentDirectory/$fileName } returnMessage
catch { exec /bin/chmod a+r $experimentDirectory/$fileName } returnMessage
catch { exec /bin/chmod a-w $experimentDirectory/$fileName } returnMessage
}
} else {
set errorMessage "Skipping copy, $fileEntry is not a dated file"
catch { exec /bin/echo $fileEntry >> $madroot/LoadFailures } returnMessage
catch { exec /bin/echo $errorMessage >> $madroot/LoadFailures } returnMessage
puts $errorMessage
}
}
} else {
puts "\nNo gif or jpeg files found"
}
if { $PSfilesFound } {
puts "\nCopying the following PostScript files into MADRIGAL..."
foreach fileEntry $PSfileList {
set fileSplit [file split $fileEntry]
set fileName [lindex $fileSplit end ]
if { [ string match \[0-9\]\[0-9\]\[0-9\]\[0-9\]\[0-9\]\[0-9\] [string range $fileName 0 5] ] } {
set YYMMDD [ string range $fileName 0 5 ]
if { [ string range $YYMMDD 0 1 ] > 50 } {
set YYYY "19[ string range $YYMMDD 0 1 ]"
} else {
set YYYY "20[ string range $YYMMDD 0 1 ]"
}
# check if experiment exists in MADRIGAL and copy PostScript file there if it does
# (have to recheck since the plots may not belong to the present experiment)
set experimentDirectory "$MADRIGALDirectory/$YYYY/$siteName/[ YYmmDDToMADRIGALDate $YYMMDD ]"
if { ![ catch { exec /bin/ls $experimentDirectory } returnMessage ] } {
# Move original data file for this experiment to the appropriate experiment directory
# gzip it, and set file permissions
if { [ catch { exec /bin/ls/ $experimentDirectory/$fileName.gz } returnMessage ] } {
# Only copy postscript file if a gzipped copy is not already present
# otherwise the gzip stage below will silently hang
puts $fileEntry
catch { exec /bin/cp $fileEntry $experimentDirectory/$fileName } returnMessage
catch { exec /bin/gzip $experimentDirectory/$fileName } returnMessage
catch { exec /bin/chmod a+r $experimentDirectory/$fileName.gz } returnMessage
catch { exec /bin/chmod a-w $experimentDirectory/$fileName.gz } returnMessage
} else {
puts "Skipping $fileEntry (gzip already exists)"
}
}
} else {
set errorMessage "Skipping copy, $fileEntry is not a dated file"
catch { exec /bin/echo $fileEntry >> $madroot/LoadFailures } returnMessage
catch { exec /bin/echo $errorMessage >> $madroot/LoadFailures } returnMessage
puts $errorMessage
}
}
} else {
puts "\nNo PostScript files found"
}
# puts "\nUpdating master tables...."
if { [ catch { exec $madroot/bin/updateMaster >@stdout } returnMessage ] } {
puts $returnMessage
}
puts $returnMessage
puts "\a\a\a\n"
puts "Please run updateMADRIGALAdditions.exp to create JPEGs, etc."
puts "Remember to clear stage directories as necessary, and check"
puts "$madroot/LoadFailures for files which did not load.\n"
puts "A log of added data is at $madroot/INDEX_of_data_added_to_SRI_MADRIGAL"
puts "\nThank you for using the automated MADRIGAL data loader!\n"
exit
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ffw_print.tcl
Type: application/x-tcl
Size: 4774 bytes
Desc: not available
Url : http://www.haystack.mit.edu/pipermail/openmadrigal-developers/attachments/20031015/6115893f/ffw_print.tcl
More information about the OpenMadrigal-developers
mailing list