'
' Small utility to get all BaCon programs over HTTP from the BaCon website
'
' Dec 2010 - PvE.
' Mar 2011: added size check - PvE
'-----------------------------------------------------------------------------------

' Omit empty results with SPLIT
OPTION COLLAPSE TRUE

' Define website
website$ = "www.basic-converter.org"

' Get the list with files
OPEN CONCAT$(website$, ":80") FOR NETWORK AS mynet
SEND CONCAT$("GET /files.txt HTTP/1.1\r\nHost: ", website$, "\r\n\r\n") TO mynet

REPEAT
    RECEIVE dat$ FROM mynet
    IF LEN(dat$) > 0 THEN total$ = CONCAT$(total$, dat$)
UNTIL ISFALSE(WAIT(mynet, 500))

' Split the text into separate filenames
SPLIT MID$(total$, INSTR(total$, "\r\n\r\n")+4) BY NL$ TO file$ SIZE amount

' Now get each individual file
FOR x = 0 TO amount-1
    PRINT "Fetching ", file$[x], "... ";
    attempt = 0

    REPEAT
        ' Keep track of attempts
        INCR attempt
        SELECT attempt
            CASE 2;
            CASE 3
                PRINT "Failed! Retrying..."
            CASE 4
                PRINT "Could not download ", file$[x], " - skipping."
                DELETE FILE file$[x]
                BREAK
        END SELECT

        SEND CONCAT$("GET /", file$[x], " HTTP/1.1\r\nHost: ", website$, "\r\n\r\n") TO mynet
        total$ = ""

        RECEIVE dat$ FROM mynet

        ' Get the filesize from the HTTP header
        dat$ = MID$(dat$, INSTR(dat$, "Content-Length:")+15)
        length = VAL(LEFT$(dat$, INSTR(dat$, NL$)))

        ' As long as there is data, get it
        WHILE WAIT(mynet, 500)
            total$ = CONCAT$(total$, dat$)
            RECEIVE dat$ FROM mynet
        WEND
        total$ = CONCAT$(total$, dat$)

        ' Write to file
        OPEN file$[x] FOR WRITING AS baconfile
            WRITELN MID$(total$, INSTR(total$, "\r\n\r\n")+4) TO baconfile
        CLOSE FILE baconfile

    ' Make sure filelength equals size mentioned in HTTP header +1 for last newline
    UNTIL FILELEN(file$[x]) = length+1

    IF attempt < 4 THEN PRINT "done."
NEXT

CLOSE NETWORK mynet