D
deskcheck1
Guest
Hi,
I have weather station text files whose data need validation. There are 17,000+ station files; each station has 42,368 lines. So I'm developing a VB.Net console application that creates a text file listing all the errors in the data.
Using this error file, I need to read the corresponding weather station text file and fix all errors in the weather station files. However, I'm getting an error, "The process cannot access the weather station file because it's being used by another process." Below is my code snippet:
Module Module 1
Dim stationPath As String = "E:\EpicWeather" ' location of epic files
Dim errorPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) ' error path
Dim errors As ArrayList = New ArrayList()
Dim errorFile = errorPath & "\" & "EpicFormatError.txt"
Sub Main()
ValidateFormat(stationPath)
CorrectErrors(errorFile)
End Sub
Private Sub ValidateFormat(stationPath As String)
stationPath = "F:\Projects2015VB\ValidateStationData\EpicWeather"
Dim files = Directory.GetFiles(stationPath)
Dim yyyy = "", mm = "", dd = ""
Dim fileCount = 0
For Each file In files
Dim lines() As String = Nothing
Dim totalLines = 0
Dim reader As FileIO.TextFieldParser = New FileIO.TextFieldParser(file)
reader.TextFieldType = FileIO.FieldType.FixedWidth
Dim day = 0
Dim station = Path.GetFileNameWithoutExtension(file)
'Count lines
lines = New StreamReader(file).ReadToEnd().Split(vbCrLf) 'totalLines = 42369
totalLines = lines.GetLength(0)
'Set reader's width based on format columns
reader.SetFieldWidths(6, 4, 4, 24)
Dim lineCount = 0
While Not reader.EndOfData
Try
For Each line In lines
'Read one line at a time
Dim currField = reader.ReadFields() ' 4 columns
If currField IsNot Nothing Then
yyyy = currField(0)
mm = currField(1)
dd = currField(2)
If CInt(dd) <= 31 Then
day = CInt(dd) + 1
End If
'Add last line to error array
If line.Length < 38 Then
Dim err = "Format Error: " & yyyy & " " & mm & " " & day & " Station " & station
err = err.Replace(vbLf, " ")
errors.Add(err)
End If
End If
lineCount = lineCount + 1
Next
Catch ex As FileIO.MalformedLineException
'Add lines to error array
Dim err = "Format Error: " & yyyy & " " & mm & " " & day & " Station " & station
err = err.Replace(vbLf, " ")
errors.Add(err)
Dim msg = ex.Message & yyyy & mm & dd & " " & station
msg = msg.Replace(vbLf, " ")
Console.WriteLine(msg)
End Try
End While
reader.Dispose()
fileCount += 1
Next
End Sub
Private Sub CorrectErrors(errorFile As String)
Dim errResult() As String = Nothing
Dim weatherResult() As String = Nothing
Dim station As String = ""
Dim totalLines As Integer = 0
Dim yr = 0, mo = 0, dy = 0
Dim yyyy = 0, mm = 0, dd = 0
Dim filePath As String = ""
Dim errDate, weatherDate As Date
Dim sol, tmax, tmin, pcp As Single
Dim count = 0
Using reader As New FileIO.TextFieldParser(errorFile)
reader.TextFieldType = FileIO.FieldType.FixedWidth
reader.SetFieldWidths(6, 7, 6, 3, 3, 9, 13)
errResult = New StreamReader(errorFile).ReadToEnd().Split(vbCrLf)
totalLines = errResult.GetLength(0)
For i = 0 To totalLines - 1
If Not i = totalLines - 1 Then
errResult(i) = errResult(i).Replace(vbLf, "")
End If
Next
Do While Not reader.EndOfData
Dim errRow = reader.ReadFields()
'Get date and station name
yr = CInt(errRow(2))
mo = CInt(errRow(3))
dy = CInt(errRow(4))
errDate = New DateTime(yr, mo, dy)
station = errRow(errRow.Length - 1)
'Get station file
filePath = stationPath & "\" & station & ".txt"
Using wReader As New FileIO.TextFieldParser(filePath)
wReader.TextFieldType = FileIO.FieldType.FixedWidth
wReader.SetFieldWidths(6, 6, 4, 4, 24)
weatherResult = New StreamReader(filePath).ReadToEnd().Split(vbCrLf)
For i = 0 To weatherResult.Length - 1
If Not i = weatherResult.Length - 1 Then
weatherResult(i) = weatherResult(i).Replace(vbLf, "")
End If
Next
For j = 0 To weatherResult.Length - 1
yyyy = Mid(weatherResult(j), 1, 6)
mm = Mid(weatherResult(j), 7, 4)
dd = Mid(weatherResult(j), 11, 4)
weatherDate = New DateTime(yyyy, mm, dd)
If errDate = weatherDate And weatherResult(j).Length < 38 And Not weatherResult(j).Contains("-") Then
sol = Mid(weatherResult(j), 15, 6)
tmax = Mid(weatherResult(j), 21, 6)
tmin = Mid(weatherResult(j), 27, 6)
pcp = Mid(weatherResult(j), 33, 6)
Dim newStr As String = Mid(weatherResult(j), 17, 24)
newStr = newStr & "0"
Dim writer = My.Computer.FileSystem.OpenTextFileWriter(filePath, False, System.Text.Encoding.ASCII) 'Error here*************
writer.WriteLine(newStr)
End If
Next
End Using
Loop
count = count + 1
End Using
End Sub
Sample Error Text File:
Format Error: 1900 03 24 Station US10cumi003
Format Error: 1917 10 20 Station US10cumi003
Format Error: 1923 05 8 Station US10cumi003
Format Error: 1926 10 23 Station US10cumi003
Format Error: 1935 02 15 Station US10cumi003
Format Error: 1937 03 7 Station US10cumi003
Format Error: 1944 11 12 Station US10cumi003
Format Error: 1954 11 15 Station US10cumi003
Format Error: 1976 11 3 Station US10cumi003
Format Error: 1986 03 27 Station US10cumi003
Format Error: 2013 01 6 Station US10cumi003
Sample Weather Station text file:
1900 03 22 0.0020.64-00.79000.00
1900 03 23 0.0016.64-00.48000.00
1900 03 24 0.0007.8100.00004.48
1900 03 25 0.0011.80000.48004.26
1900 03 26 0.0010.92-02.09000.00
1917 10 18 0.0010.09000.39000.07
1917 10 19 0.0008.26-10.39000.01
1917 10 20 0.0008.0500.00000.57
1917 10 21 0.0015.90-05.04000.00
1917 10 22 0.0012.02-00.91000.01
1986 03 26 0.0015.82002.15000.00
1986 03 27 0.0015.7800.00000.00
1986 03 28 0.0025.82005.02000.00
2013 01 05 0.0-00.18-11.86000.00
2013 01 06 0.000.00-13.83000.00
2013 01 07 0.0000.54-12.43000.00
Marilyn Gambone
Continue reading...
I have weather station text files whose data need validation. There are 17,000+ station files; each station has 42,368 lines. So I'm developing a VB.Net console application that creates a text file listing all the errors in the data.
Using this error file, I need to read the corresponding weather station text file and fix all errors in the weather station files. However, I'm getting an error, "The process cannot access the weather station file because it's being used by another process." Below is my code snippet:
Module Module 1
Dim stationPath As String = "E:\EpicWeather" ' location of epic files
Dim errorPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) ' error path
Dim errors As ArrayList = New ArrayList()
Dim errorFile = errorPath & "\" & "EpicFormatError.txt"
Sub Main()
ValidateFormat(stationPath)
CorrectErrors(errorFile)
End Sub
Private Sub ValidateFormat(stationPath As String)
stationPath = "F:\Projects2015VB\ValidateStationData\EpicWeather"
Dim files = Directory.GetFiles(stationPath)
Dim yyyy = "", mm = "", dd = ""
Dim fileCount = 0
For Each file In files
Dim lines() As String = Nothing
Dim totalLines = 0
Dim reader As FileIO.TextFieldParser = New FileIO.TextFieldParser(file)
reader.TextFieldType = FileIO.FieldType.FixedWidth
Dim day = 0
Dim station = Path.GetFileNameWithoutExtension(file)
'Count lines
lines = New StreamReader(file).ReadToEnd().Split(vbCrLf) 'totalLines = 42369
totalLines = lines.GetLength(0)
'Set reader's width based on format columns
reader.SetFieldWidths(6, 4, 4, 24)
Dim lineCount = 0
While Not reader.EndOfData
Try
For Each line In lines
'Read one line at a time
Dim currField = reader.ReadFields() ' 4 columns
If currField IsNot Nothing Then
yyyy = currField(0)
mm = currField(1)
dd = currField(2)
If CInt(dd) <= 31 Then
day = CInt(dd) + 1
End If
'Add last line to error array
If line.Length < 38 Then
Dim err = "Format Error: " & yyyy & " " & mm & " " & day & " Station " & station
err = err.Replace(vbLf, " ")
errors.Add(err)
End If
End If
lineCount = lineCount + 1
Next
Catch ex As FileIO.MalformedLineException
'Add lines to error array
Dim err = "Format Error: " & yyyy & " " & mm & " " & day & " Station " & station
err = err.Replace(vbLf, " ")
errors.Add(err)
Dim msg = ex.Message & yyyy & mm & dd & " " & station
msg = msg.Replace(vbLf, " ")
Console.WriteLine(msg)
End Try
End While
reader.Dispose()
fileCount += 1
Next
End Sub
Private Sub CorrectErrors(errorFile As String)
Dim errResult() As String = Nothing
Dim weatherResult() As String = Nothing
Dim station As String = ""
Dim totalLines As Integer = 0
Dim yr = 0, mo = 0, dy = 0
Dim yyyy = 0, mm = 0, dd = 0
Dim filePath As String = ""
Dim errDate, weatherDate As Date
Dim sol, tmax, tmin, pcp As Single
Dim count = 0
Using reader As New FileIO.TextFieldParser(errorFile)
reader.TextFieldType = FileIO.FieldType.FixedWidth
reader.SetFieldWidths(6, 7, 6, 3, 3, 9, 13)
errResult = New StreamReader(errorFile).ReadToEnd().Split(vbCrLf)
totalLines = errResult.GetLength(0)
For i = 0 To totalLines - 1
If Not i = totalLines - 1 Then
errResult(i) = errResult(i).Replace(vbLf, "")
End If
Next
Do While Not reader.EndOfData
Dim errRow = reader.ReadFields()
'Get date and station name
yr = CInt(errRow(2))
mo = CInt(errRow(3))
dy = CInt(errRow(4))
errDate = New DateTime(yr, mo, dy)
station = errRow(errRow.Length - 1)
'Get station file
filePath = stationPath & "\" & station & ".txt"
Using wReader As New FileIO.TextFieldParser(filePath)
wReader.TextFieldType = FileIO.FieldType.FixedWidth
wReader.SetFieldWidths(6, 6, 4, 4, 24)
weatherResult = New StreamReader(filePath).ReadToEnd().Split(vbCrLf)
For i = 0 To weatherResult.Length - 1
If Not i = weatherResult.Length - 1 Then
weatherResult(i) = weatherResult(i).Replace(vbLf, "")
End If
Next
For j = 0 To weatherResult.Length - 1
yyyy = Mid(weatherResult(j), 1, 6)
mm = Mid(weatherResult(j), 7, 4)
dd = Mid(weatherResult(j), 11, 4)
weatherDate = New DateTime(yyyy, mm, dd)
If errDate = weatherDate And weatherResult(j).Length < 38 And Not weatherResult(j).Contains("-") Then
sol = Mid(weatherResult(j), 15, 6)
tmax = Mid(weatherResult(j), 21, 6)
tmin = Mid(weatherResult(j), 27, 6)
pcp = Mid(weatherResult(j), 33, 6)
Dim newStr As String = Mid(weatherResult(j), 17, 24)
newStr = newStr & "0"
Dim writer = My.Computer.FileSystem.OpenTextFileWriter(filePath, False, System.Text.Encoding.ASCII) 'Error here*************
writer.WriteLine(newStr)
End If
Next
End Using
Loop
count = count + 1
End Using
End Sub
Sample Error Text File:
Format Error: 1900 03 24 Station US10cumi003
Format Error: 1917 10 20 Station US10cumi003
Format Error: 1923 05 8 Station US10cumi003
Format Error: 1926 10 23 Station US10cumi003
Format Error: 1935 02 15 Station US10cumi003
Format Error: 1937 03 7 Station US10cumi003
Format Error: 1944 11 12 Station US10cumi003
Format Error: 1954 11 15 Station US10cumi003
Format Error: 1976 11 3 Station US10cumi003
Format Error: 1986 03 27 Station US10cumi003
Format Error: 2013 01 6 Station US10cumi003
Sample Weather Station text file:
1900 03 22 0.0020.64-00.79000.00
1900 03 23 0.0016.64-00.48000.00
1900 03 24 0.0007.8100.00004.48
1900 03 25 0.0011.80000.48004.26
1900 03 26 0.0010.92-02.09000.00
1917 10 18 0.0010.09000.39000.07
1917 10 19 0.0008.26-10.39000.01
1917 10 20 0.0008.0500.00000.57
1917 10 21 0.0015.90-05.04000.00
1917 10 22 0.0012.02-00.91000.01
1986 03 26 0.0015.82002.15000.00
1986 03 27 0.0015.7800.00000.00
1986 03 28 0.0025.82005.02000.00
2013 01 05 0.0-00.18-11.86000.00
2013 01 06 0.000.00-13.83000.00
2013 01 07 0.0000.54-12.43000.00
Marilyn Gambone
Continue reading...