web form filling emulation

[Merrion]

I have written an app that "fills in" a web form and posts it.
This is done by encoding the parameters in multipart/form-data mode e.g.:-

Public Class multipart_formdata

#Region "Private constants"
    Private Const DEF_BOUNDARY As String = "-----------------------------7d6386366801f6"
#End Region

#Region "Private members"
    Private _Boundary As String
    Private _FormText As New StringBuilder
#End Region

#Region "Public interface"
    Public ReadOnly Property MultipartBoundary() As String
        Get
            Return _Boundary
        End Get
    End Property

    Public ReadOnly Property Text() As String
        Get
            Return _FormText.ToString & " " & _Boundary & "--  "
        End Get
    End Property

    Public Sub AddParameter(ByVal ParameterName As String, ByVal ParameterValue As String)

        _FormText.Append(_Boundary) ----------------------------7ce3023980c 
        _FormText.Append(" ")
        _FormText.Append("Content-Disposition: form-data; name=")
        _FormText.Append(QuoteString(ParameterName))
        _FormText.Append(" ")
        _FormText.Append(ParameterValue)
        _FormText.Append(" ")
        Content-Disposition: form-data; name="UploadFormName"; filename="C:\Directory\UploadFile.txt" 
        Content-Type: text/plain 

        This is a test file for ProjectUpload.
        -----------------------------7ce3023980c-- 

    End Sub
#End Region

#Region "Public constructor"
    Public Sub New()
        Me.New(DEF_BOUNDARY)
    End Sub

    Public Sub New(ByVal Boundary As String)

        If Boundary Is Nothing OrElse Boundary = "" Then
            _Boundary = DEF_BOUNDARY
        Else
            _Boundary = Boundary
        End If

    End Sub
#End Region

#Region "Private methods"
    Private Function QuoteString(ByVal sIn As String) As String
        Return Chr(34) & sIn & Chr(34)
    End Function
#End Region

End Class

Then , for example to send a usename and password to a URL you could do thus:-

            Dim webReq As HttpWebRequest = CType(WebRequest.Create("http://www.test.com"), HttpWebRequest)
            Dim byteOut() As Byte
            Dim _form As New multipart_formdata
            With _form
                .AddParameter("username", _Username)
                .AddParameter("password", _Password)
            End With

            Dim encoder As New System.Text.UTF8Encoding
            byteOut = encoder.GetBytes(_form.Text)
            With webReq
                .CachePolicy = New System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache)
                .Accept = "*/*"
                .Method = "POST" set the request method to POST
                .ContentType = "multipart/form-data; boundary=" & _form.MultipartBoundary
                .ContentLength = byteOut.Length - 1
                .Expect = Nothing
                .Referer = "http://localhost/PresentValueServlet/getValuation.htm"
            End With
            Dim outStream As Stream
            outStream = webReq.GetRequestStream()
            If outStream.CanWrite Then
                        \\ write the request string to it...
                        outStream.Write(byteOut, 0, byteOut.Length - 1)
                        outStream.Close()
             End If

The problem is that even though the form data is sent to the web service, the Page.Form property is blank at the recieving end - any ideas?

 

[Gill Bates]

Unless you plan to upload a file I would not use "multipart/form-data" for the content type. You should use "application/x-www-form-urlencoded"" instead. This is the way I usually do it:

StringBuilder builder = new StringBuilder();
builder.Append("name=");
builder.Append(System.Web.HttpUtility.UrlEncode("Gill Bates"));
builder.Append("&country=");
builder.Append(System.Web.HttpUtility.UrlEncode("United States"));

byte[] data = Encoding.ASCII.GetBytes(builder.ToString());

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(new Uri("http://localhost:4087/post_test/form.aspx"));
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = data.Length;

Stream stream = request.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();

WebResponse response = request.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream());
Console.WriteLine(reader.ReadToEnd());
reader.Close();
response.GetResponseStream().Close();

 

[Merrion]

Unfortunately I do need to upload a file - the web site that Im automating takes a username, password and file to upload...

I have written an ASP page to emulate it thus:-

    Protected Sub Page_Load(ByVal sender As Object, ByVal e As System.EventArgs) Handles Me.Load

        Dim RequestOK As Boolean
        Dim input(Request.TotalBytes) As Byte
        Request.InputStream.Read(input, 0, Request.TotalBytes)

        Dont send a response until the input is processed...
        Page.Response.BufferOutput = True
        Page.Response.ContentType = "application/xml"

        With Request.Form
            If .Item("username") = "Merrion" Then
                If .Item("password") = "Duncan" Then
                        If Page.Request.Files.Count > 0 Then
                            RequestOK = True
                            Page.Request.Files(0).SaveAs("c:\temp\uploaded_file.xml")
                        End If
                End If
            End If
            If Not RequestOK Then
                Dim ErrOutput As New XmlTextWriter(Page.Response.OutputStream, System.Text.Encoding.UTF8)
                With ErrOutput
                    .WriteStartDocument()
                    .WriteStartElement("Return")
                    .WriteStartElement("Settings")
                    .WriteElementString("Method", Request.HttpMethod)
                    For h As Integer = 0 To Request.Headers.Count - 1
                        .WriteElementString(Request.Headers.Keys(h), Request.Headers.Item(h))
                    Next
                    .WriteEndElement()
                    .WriteStartElement("DataContent")
                    If input.Length = 0 Then
                        .WriteElementString("data", "Not set")
                    Else
                        Dim foo As New System.Text.ASCIIEncoding
                        .WriteElementString("data", foo.GetChars(input, 0, input.Length))
                    End If
                    .WriteEndElement()
                    .WriteStartElement("Errors")
                    .WriteElementString("FormValues", Request.Form.Count.ToString)
                    For n As Integer = 0 To Request.Form.Count - 1
                        .WriteElementString(Request.Form.Keys.Item(n), Request.Form.Item(n))
                    Next
                    .WriteEndElement() Errors
                    .WriteEndElement() Return 
                    .WriteEndDocument()
                    .Close()
                End With
            End If
        End With


    End Sub

But when I write to it from my app (as per the code above) it has no values in the Page.Forms collection - i.e. the return is:-

<?xml version="1.0" encoding="utf-8"?>
<Return>
<Settings>
<Method>POST</Method>
<Connection>Keep-Alive</Connection>
<Content-Length>364</Content-Length>
<Content-Type>multipart/form-data; boundary=-----------------------------7d6386366801f6</Content-Type>
<Accept>application/xml</Accept>
<Expect>100-continue</Expect>
<Host>localhost</Host>
</Settings>
<DataContent>
<data>-----------------------------7d6386366801f6
Content-Disposition: form-data; name="username"

Merrion
-----------------------------7d6386366801f6
Content-Disposition: form-data; name="password"

testpwd
-----------------------------7d6386366801f6
Content-Disposition: form-data; name="asof"

04Jul06
-----------------------------7d6386366801f6--

</data>
</DataContent>
<Errors>
<FormValues>0</FormValues>
</Errors>
</Return>

but when I use an HTML form to send the data I get:-

 <?xml version="1.0" encoding="utf-8" ?> 
- <Return>
- <Settings>
  <Method>POST</Method> 
  <Cache-Control>no-cache</Cache-Control> 
  <Connection>Keep-Alive</Connection> 
  <Content-Length>350</Content-Length> 
  <Content-Type>multipart/form-data; boundary=---------------------------7d61021b7503f8</Content-Type> 
  <Accept>*/*</Accept> 
  <Accept-Encoding>gzip, deflate</Accept-Encoding> 
  <Accept-Language>en-ie</Accept-Language> 
  <Host>localhost</Host> 
  <Referer>http://localhost/PresentValueServlet/getValuation.htm</Referer> 
  <User-Agent>Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705; .NET CLR 2.0.50727)</User-Agent> 
  </Settings>
- <DataContent>
  <data>-----------------------------7d61021b7503f8 
Content-Disposition: form-data; name="username" 

Merrion 
-----------------------------7d61021b7503f8 
Content-Disposition: form-data; name="password"

Duncan 
-----------------------------7d61021b7503f8 
Content-Disposition: form-data; name="asof" 
-----------------------------7d61021b7503f8-- 
</data> 
  </DataContent>
- <Errors>
  <FormValues>3</FormValues> 
  <username>Merrion</username> 
  <password>Duncan</password> 
  <asof /> 
  </Errors>
  </Return>