From 87a234b45b2a118f67f6d4147fce9a1fadd27f47 Mon Sep 17 00:00:00 2001 From: Roland Heyder Date: Tue, 22 Nov 2022 18:20:48 +0100 Subject: [PATCH] Wiki download changes Using built-in TCP/IP and HTTP protocol via manual request headers. --- source/global/IDEsettings.bas | 5 +- source/ide/wiki/wiki_methods.bas | 99 ++++++++++++++++++++++++-------- 2 files changed, 79 insertions(+), 25 deletions(-) diff --git a/source/global/IDEsettings.bas b/source/global/IDEsettings.bas index faca26d15..b4e8bd908 100644 --- a/source/global/IDEsettings.bas +++ b/source/global/IDEsettings.bas @@ -226,10 +226,11 @@ IF result = 0 OR idedebuginfo <> -1 THEN END IF Include_GDB_Debugging_Info = idedebuginfo -wikiBaseAddress$ = "https://qb64phoenix.com/qb64wiki" +wikiBaseAddress$ = "http://qb64phoenix.com/qb64wiki" IF ReadConfigSetting(generalSettingsSection$, "WikiBaseAddress", value$) THEN wikiBaseAddress$ = value$ -ELSE WriteConfigSetting generalSettingsSection$, "WikiBaseAddress", wikiBaseAddress$ +ELSE + WriteConfigSetting generalSettingsSection$, "WikiBaseAddress", wikiBaseAddress$ END IF UseGuiDialogs = ReadWriteBooleanSettingValue%(generalSettingsSection$, "UseGuiDialogs", -1) diff --git a/source/ide/wiki/wiki_methods.bas b/source/ide/wiki/wiki_methods.bas index e9eb1f0bc..a9b247089 100644 --- a/source/ide/wiki/wiki_methods.bas +++ b/source/ide/wiki/wiki_methods.bas @@ -38,22 +38,22 @@ FUNCTION Wiki$ (PageName$) 'Read cached wiki page (download, if not yet cached) END IF 'Check for curl - IF _SHELLHIDE("curl --version >NUL") <> 0 THEN - a$ = CHR$(10) + "{{PageInternalError}}" + CHR$(10) - IF PageName$ = "Initialize" THEN - a$ = a$ + "To be able to initialize the help system, " - ELSEIF PageName$ = "Update All" THEN - a$ = a$ + "To be able to update the help pages from the online Wiki, " - ELSE - a$ = a$ + "The requested help page is not yet cached locally. To download the help page from the online Wiki, " - END IF - a$ = a$ + "a tool called ''curl'' is required, but it wasn't found on your system." + CHR$(10) + CHR$(10) - a$ = a$ + "* To get ''curl'', visit the official [https://curl.se/download.html download page]." + CHR$(10) - a$ = a$ + "** Grab the latest ''binary'' archive available for your system." + CHR$(10) - a$ = a$ + "** Unpack and drop the ''curl'' executable into the '''qb64pe''' folder." + CHR$(10) - a$ = a$ + "** If there's a file named ''curl-ca-bundle.crt'' or similar, drop it into the '''qb64pe''' folder too." + CHR$(10) - Wiki$ = a$: EXIT FUNCTION - END IF + 'IF _SHELLHIDE("curl --version >NUL") <> 0 THEN + ' a$ = CHR$(10) + "{{PageInternalError}}" + CHR$(10) + ' IF PageName$ = "Initialize" THEN + ' a$ = a$ + "To be able to initialize the help system, " + ' ELSEIF PageName$ = "Update All" THEN + ' a$ = a$ + "To be able to update the help pages from the online Wiki, " + ' ELSE + ' a$ = a$ + "The requested help page is not yet cached locally. To download the help page from the online Wiki, " + ' END IF + ' a$ = a$ + "a tool called ''curl'' is required, but it wasn't found on your system." + CHR$(10) + CHR$(10) + ' a$ = a$ + "* To get ''curl'', visit the official [https://curl.se/download.html download page]." + CHR$(10) + ' a$ = a$ + "** Grab the latest ''binary'' archive available for your system." + CHR$(10) + ' a$ = a$ + "** Unpack and drop the ''curl'' executable into the '''qb64pe''' folder." + CHR$(10) + ' a$ = a$ + "** If there's a file named ''curl-ca-bundle.crt'' or similar, drop it into the '''qb64pe''' folder too." + CHR$(10) + ' Wiki$ = a$: EXIT FUNCTION + 'END IF 'Download message (Status Bar) IF Help_Recaching = 0 THEN @@ -68,19 +68,23 @@ FUNCTION Wiki$ (PageName$) 'Read cached wiki page (download, if not yet cached) END IF 'Url query and output arguments for curl - url$ = CHR$(34) + wikiBaseAddress$ + "/index.php?title=" + PageName2$ + "&action=edit" + CHR$(34) + 'url$ = CHR$(34) + wikiBaseAddress$ + "/index.php?title=" + PageName2$ + "&action=edit" + CHR$(34) + url$ = wikiBaseAddress$ + "/index.php?title=" + PageName2$ + "&action=edit" + IF LCASE$(LEFT$(url$, 8)) = "https://" THEN url$ = MID$(url$, 9) + IF LCASE$(LEFT$(url$, 7)) = "http://" THEN url$ = MID$(url$, 8) outputFile$ = Cache_Folder$ + "/" + PageName3$ + ".txt" 'Wikitext delimiters s1$ = "name=" + CHR$(34) + "wpTextbox1" + CHR$(34) + ">" s2$ = "" 'Download page using curl - SHELL _HIDE "curl --silent -o " + CHR$(34) + outputFile$ + CHR$(34) + " " + url$ + 'SHELL _HIDE "curl --silent -o " + CHR$(34) + outputFile$ + CHR$(34) + " " + url$ fh = FREEFILE - OPEN outputFile$ FOR BINARY AS #fh 'get new content - a$ = SPACE$(LOF(fh)) - GET #fh, 1, a$ - CLOSE #fh + 'OPEN outputFile$ FOR BINARY AS #fh 'get new content + 'a$ = SPACE$(LOF(fh)) + 'GET #fh, 1, a$ + 'CLOSE #fh + a$ = wikiDLPage$(url$, 15) 'Find wikitext in the downloaded page s1 = INSTR(a$, s1$) @@ -121,7 +125,7 @@ FUNCTION Wiki$ (PageName$) 'Read cached wiki page (download, if not yet cached) CLOSE #fh ELSE 'Delete page, if empty or corrupted (force re-download on next access) - KILL outputFile$ + 'KILL outputFile$ a$ = CHR$(10) + "{{PageInternalError}}" + CHR$(10) +_ "* Either the requested page is not yet available in the Wiki," + CHR$(10) +_ "* or the download from Wiki failed and corrupted the page data." + CHR$(10) +_ @@ -1068,6 +1072,55 @@ FUNCTION wikiSafeName$ (page$) 'create a unique name for both case sensitive & i wikiSafeName$ = page$ + "_" + ext$ END FUNCTION +FUNCTION wikiDLPage$ (url$, timeout!) +'--- prepare args --- +spo% = INSTR(url$, "/") +hst$ = LEFT$(url$, spo% - 1) +obj$ = MID$(url$, spo%) +eol$ = CHR$(13) + CHR$(10) +wikiDLPage$ = "" +'--- open client --- +ch& = _OPENCLIENT("TCP/IP:80:" + hst$) +IF ch& = 0 THEN EXIT FUNCTION +'--- send request --- +req$ = "GET " + obj$ + " HTTP/1.1" + eol$ +req$ = req$ + "Host: " + hst$ + eol$ +req$ = req$ + "User-Agent: QB64-PE/" + Version$ + " (qb64phoenix.com)" + eol$ +req$ = req$ + "Cache-Control: no-cache" + eol$ +req$ = req$ + eol$ +PUT ch&, , req$ +res$ = "" +'--- wait for response --- +st! = TIMER +DO + _DELAY 0.05 + GET ch&, , rec$ + IF LEN(rec$) > 0 THEN st! = TIMER + res$ = res$ + rec$ + IF LEN(res$) < 15 OR LEFT$(res$, 15) = "HTTP/1.1 200 OK" THEN + cl% = INSTR(res$, "Content-Length:") + IF cl% > 0 THEN + cle% = INSTR(cl%, res$, eol$) + IF cle% > 0 THEN + le& = VAL(MID$(res$, cl% + 15, cle% - cl% - 14)) + das% = INSTR(cle%, res$, eol$ + eol$) + IF das% > 0 THEN + das% = das% + 4 + IF (LEN(res$) - das% + 1) = le& THEN + CLOSE ch& + wikiDLPage$ = MID$(res$, das%, le&) + EXIT FUNCTION + END IF + END IF + END IF + END IF + ELSE + timeout! = 0 + END IF +LOOP UNTIL TIMER > st! + timeout! +CLOSE ch& +END FUNCTION + FUNCTION wikiLookAhead$ (a$, i, token$) 'Prefetch further wiki text wikiLookAhead$ = "": IF i >= LEN(a$) THEN EXIT FUNCTION j = INSTR(i, a$, token$)