fetch.bac

'
' Small utility to get all BaCon programs over HTTP from the BaCon website
'
' Dec 2010 - PvE.
' Mar 2011: added size check - PvE
' Jul 2013: various improvements, network handling more robust - PvE
' January 2017: option collapse to skip empty results - PvE
'-----------------------------------------------------------------------------------
OPTION COLLAPSE TRUE

CONST Sep$ = CHR$(13) & NL$ & CHR$(13) & NL$

' Define website
website$ = "www.basic-converter.org"

' Get the list with files
OPEN website$ & ":80" FOR NETWORK AS mynet
    SEND "GET /files.txt HTTP/1.1\r\nHost: " & website$ & Sep$ TO mynet

    REPEAT
        RECEIVE dat$ FROM mynet
        IF LEN(dat$) > 0 THEN total$ = total$ & dat$
    UNTIL ISFALSE(WAIT(mynet, 500))
CLOSE NETWORK mynet

' Split the text into separate filenames
FOR file$ IN MID$(total$, INSTR(total$, Sep$)+4) STEP NL$

    PRINT "Fetching ", file$, "... ";
    attempt = 0

    REPEAT
        ' We create a new connection for each individual file
        OPEN website$ & ":80" FOR NETWORK AS mynet

        ' Get the current file
        SEND "GET /" & file$ & " HTTP/1.1\r\nHost: " & website$ & Sep$ TO mynet
        total$ = ""

        ' Prevent endless waiting for server
        IF NOT(WAIT(mynet, 250)) THEN
            COLOR FG TO CYAN
            PRINT "Timeout on server! Retrying... ";
            SLEEP 500
            COLOR RESET
            CLOSE NETWORK mynet
            INCR attempt
            IF attempt = 4 THEN
                COLOR FG TO RED
                PRINT "Could not download ", file$, " - skipping."
                SLEEP 500
                COLOR RESET
                BREAK
            END IF
            CONTINUE
        ELSE
            ' So there is data
            RECEIVE dat$ FROM mynet

            ' Get the filesize from the HTTP header
            dat$ = MID$(dat$, INSTR(dat$, "Content-Length:")+15)
            length = VAL(LEFT$(dat$, INSTR(dat$, NL$)))

            ' As long as there is data, get it
            WHILE WAIT(mynet, 500) AND LEN(dat$) > 0
                total$ = total$ & dat$
                RECEIVE dat$ FROM mynet
            WEND

            total$ = total$ & dat$

            ' Write to file
            OPEN file$ FOR WRITING AS baconfile
                WRITELN MID$(total$, INSTR(total$, Sep$)+4) TO baconfile
            CLOSE FILE baconfile

            ' Closing the connection again
            CLOSE NETWORK mynet
        END IF

        ' If the size of the file is not correct
        IF FILELEN(file$) <> length+1 THEN

            ' Keep track of attempts
            INCR attempt

            SELECT attempt
                CASE 2;
                CASE 3
                    COLOR FG TO CYAN
                    PRINT "Failed! Retrying... ";
                    SLEEP 500
                    COLOR RESET
                CASE 4
                    COLOR FG TO RED
                    PRINT "Could not download ", file$, " - skipping."
                    SLEEP 500
                    COLOR RESET
                    IF FILEEXISTS(file$) THEN DELETE FILE file$
                    BREAK
            END SELECT
        END IF

    ' Make sure filelength equals size mentioned in HTTP header +1 for last newline
    UNTIL LEN(total$) > 0 AND FILELEN(file$) = length+1

    IF attempt < 4 THEN
        COLOR FG TO GREEN
        PRINT "done."
        COLOR RESET
    END IF
    
    ' Give the remote server some time
    SLEEP 250
NEXT



Generated by GNU Enscript 1.6.5.90.