为什么GET源码会丢失部分
本帖最后由 heavenm 于 2016-4-21 02:50 编辑#Include <Array.au3>
#include <winhttp.au3>
$_URL = 'http://list.jd.com/list.html?cat=670,677,687&ev=exbrand_18140&page=1&delivery=1&go=0&JL=6_0_0'
$_TempHtml = _WinHTTP_GetRespond(-1, $_URL, 2)
_ArrayDisplay($_TempHtml)
_ClipBoard_SetData($_TempHtml)
#include-once
#include <winhttp.au3>
Global Const $tagWINHTTP_PROXY_INFO = "DWORDdwAccessType;ptr lpszProxy;ptr lpszProxyBypass;"
;~#FUNCTION# ;===============================================================================
;~函数名...........: _WinHTTP_GetRespond
;~描述.............: 完整的HTTP传输过程
;~函数用法.........: _WinHTTP_GetRespond($hRequest, $ConnectURL [, Mode[，$hTimeOut[，$Context[，$Cookies[，$MoreHeader[，$ProxyServer[, $FuncName[, $ParamArray]]]]]]]])
;~
;~参数解释.........: $hRequest - 由 _WinHttpOpen() 函数返回的句柄 或 字符串 (Agent 的描述，也可使用-1,Dafult来创建默认描述)
;~ $ConnectURL - 请求的URL地址，其形式可为 http://UserName:UserPassWord@127.0.0.1:8181/winhttp_getrespondUpdate/
;~ $Mode - 本函数定义的模式:
;~ 0 GET (默认)
;~ 1 POST
;~ 2 返回服务器返回的内容(禁用检测的话，+256)
;~ 4 返回完整的Header
;~ 8 禁用重定向
;~ 16 禁用SSL的证书检测
;~ 32 使用自定义函数后立即返回
;~ 64 返回Connection句柄(即不关闭Connection句柄)
;~ 128访问地址为'' + 'http://' 的形式(即使用Connection句柄作为当期连接)
;~ 256返回内容绝对为二进制
;~ 512正则解析网址
;~ $hTimeOut - 超时MS(正整数,或一维四元数组)
;~ $Context - 请求需要发送的内容，一般用于POST中 (字符串)
;~ $Cookies - 指定文件头 Cookies (字符串)
;~ $MoreHeader - 指定更多的文件头信息 (单一字符串,字符串内可包含回车符)
;~ $ProxyServer - 代理服务器地址 127.0.0.1:8181 (字符串)
;~ $FuncName - 用户自定义过程名称 (执行时机为确定请求有效后，优先于本过程的文件头及数据获取过程)
;~ $ParamArray - 用户自定义过程所需参数构成的数组
;~
;~ 函数返回值......: 如果成功的话 - 返回一维8元的数组
;~ 0 - 数据内容 ( 二进制的字符串或字符串，要强制使用二进制模式，请使用256模式，并根据需要使用 Binary() 或 BinarytoString() 函数 )
;~ 1 - 服务器状态
;~ 2 - 文件长度
;~ 3 - Cookies
;~ 4 - 重定向 （该重定向为完整的URL地址)
;~ 5 - 编码类型
;~ 6 - 完整Header
;~ 7 - Connection 句柄
;~ (无内容时，上述任何一个元素均可能为-1)
;~ 如果失败的话 - 设置@error，返回最后的错误代码:
;~ @Error:
;~ 1 - 打开请求失败
;~ 2 - 打开请求失败或无法设置重定向
;~ 4 - 无法发送请求
;~ 5 - 服务器无回应或超时或使用SSL验证
;~作者 ............: Republican
;~注意点 ..........: 如果Header为多行数据，请使用@CRLF为每行非开
;~相关函数 ........: Winhttp.Au3
;~链接 ............: http://msdn.microsoft.com/en-us/library/aa384087(VS.85).aspx
;~有无例子 ........: Yes
;~日期 ............: 2011.2.19
Func _WinHTTP_GetRespond($hOpen, $ConnectURL, $Mode = Default, $hTimeOut = Default, $Context = Default, $Cookies = Default, $MoreHeader = Default, $ProxyServer= Default, $FuncName = Default, $ParamArray = -1)
; ==== 判断基本连接信息 ====
If $hOpen = "" Or $ConnectURL = "" Then Return SetError(1,0,-1)
;~ MsgBox(0,"",$ConnectURL)
; ==== 非句柄时，由函数创建句柄 ====
Local $SimpleMode = 0
If IsString($hOpen) Then
$SimpleMode = 1
$hOpen = _WinHttpOpen($hOpen)
ElseIf $hOpen = -1 Or $hOpen = Default Then
$SimpleMode = 1
$hOpen = _WinHttpOpen()
EndIf
; ==== 分解模式 $Mode 设定====
If $Mode = Default Or $Mode = -1 Then $Mode = 0
$Mode = _NumberToBinary_Winhttp($Mode)
If Not IsArray($Mode) Then Return SetError(-1,0,-1)
; ==== 提取网址中包含的句柄 ====
If $Mode = 1 Then
Local $Distinguish = StringRegExp($ConnectURL,'(.*?)(http.*)',3)
If IsArray($Distinguish) Then
$hConnect = $Distinguish
$ConnectURL = $Distinguish
EndIf
EndIf
; ==== 拆分网址 ====
$ConnectURL=_WinHttpCrackUrl($ConnectURL,$ICU_DECODE)
If @error Or Not IsArray($ConnectURL) Then Return SetError(1,0,_GetLastError_Winhttp())
; ==== 重新定义变量，方便阅读 ====
Local $ServerHost = $ConnectURL
Local $Port=$ConnectURL
Local $ObjectFile=$ConnectURL&$ConnectURL
Local $sFlags = Default
If $ConnectURL = 2 Then $sFlags = $WINHTTP_FLAG_SECURE ;=====判断是否使用SSL连接
; ==== 代理服务器设定 ====
Local $PROXY_CREDITS
If $ProxyServer <> -1 And $ProxyServer <> Default Then
$PROXY_CREDITS = StringRegExp($ProxyServer,'(.*?):(.*?)@',3)
$ProxyServer = StringRegExpReplace($ProxyServer,'(.*?)@',"")
$tProxyInfo = _WinHttpProxyInfoCreate($WINHTTP_ACCESS_TYPE_NAMED_PROXY, $ProxyServer, "localhost")
_WinHttpSetOption($hOpen, $WINHTTP_OPTION_PROXY, $tProxyInfo)
EndIf
; ==== 使用句柄创建连接(128模式下使用) ====
Switch $Mode
Case 0
$hConnect=_WinHttpConnect($hOpen, $ServerHost,$Port)
EndSwitch
; ==== 创建连接 ====
Switch $Mode
Case 0
$hRequest = _WinHttpOpenRequest($hConnect, "GET", $ObjectFile,Default,Default,Default,$sFlags)
If @error Or $hRequest = 0 Then Return SetError(2,0,_GetLastError_Winhttp())
Case 1
$hRequest = _WinHttpOpenRequest($hConnect, "POST", $ObjectFile,Default,Default,Default,$sFlags)
If @error Or $hRequest = 0Then Return SetError(2,0,_GetLastError_Winhttp())
EndSwitch
; === 代理服务器身份验证 ====
If IsArray($PROXY_CREDITS) And UBound($PROXY_CREDITS) = 2 Then
_WinHttpSetOption($hRequest, $WINHTTP_OPTION_PROXY_USERNAME, $PROXY_CREDITS)
_WinHttpSetOption($hRequest, $WINHTTP_OPTION_PROXY_PASSWORD, $PROXY_CREDITS)
EndIf
; ==== 关闭SSL中证书的验证 ====
If $Mode = 1 Then
_WinHttpSetOption($hRequest,$WINHTTP_OPTION_SECURITY_FLAGS,$SECURITY_FLAG_IGNORE_UNKNOWN_CA) ;使用非权威机构签发的证书
_WinHttpSetOption($hRequest,$WINHTTP_OPTION_SECURITY_FLAGS,$SECURITY_FLAG_IGNORE_CERT_CN_INVALID) ;使用公共名称无效的证书
_WinHttpSetOption($hRequest,$WINHTTP_OPTION_SECURITY_FLAGS,$SECURITY_FLAG_IGNORE_CERT_DATE_INVALID) ;使用过期的证书
;~ _WinHttpSetOption($hRequest,$WINHTTP_OPTION_SECURITY_FLAGS,$SECURITY_FLAG_IGNORE_CERT_WRONG_USAGE) ;使用非服务器颁发的证书
EndIf
; ==== 设置超时 ====
If $hTimeOut <> -2 And $hTimeOut <> Default And $hTimeOut <> "" Then
If IsArray($hTimeOut) And UBound($hTimeOut) = 4 Then
MsgBox(0,"",$hTimeOut)
_WinHttpSetTimeouts($hRequest,$hTimeOut,$hTimeOut,$hTimeOut,$hTimeOut)
ElseIf IsNumber($hTimeOut) Then
_WinHttpSetTimeouts($hRequest,$hTimeOut,$hTimeOut,$hTimeOut,$hTimeOut)
EndIf
EndIf
; ==== 登录需要用户验证的HTTP服务器 ====
If $ConnectURL <> "" Or $ConnectURL <> "" Then _WinHttpSetCredentials($hRequest, $WINHTTP_AUTH_TARGET_SERVER, $WINHTTP_AUTH_SCHEME_BASIC, $ConnectURL, $ConnectURL)
; ==== 禁止重定向 ====
If $Mode = 1 Then _WinHttpSetOption($hRequest,$WINHTTP_OPTION_DISABLE_FEATURE,$WINHTTP_DISABLE_REDIRECTS)
If @error Then Return SetError(3,0,_GetLastError_Winhttp())
; ==== Header构建 ====
Local $BinaryMode=0
If $Context <> -1 And $Context <> Default And $Context <> "" Then
Switch IsString($Context)
Case 1
_WinHttpAddRequestHeaders($hRequest, "Content-Length: "&StringLen($ConText)&@CRLF)
Case 0
_WinHttpAddRequestHeaders($hRequest, "Content-Length: "&BinaryLen($ConText)&@CRLF)
$BinaryMode = 1
EndSwitch
EndIf
If $Cookies <> -1 And $Cookies <> Default And $Cookies <> "" Then _WinHttpAddRequestHeaders($hRequest, "Cookie: "&$Cookies&@CRLF)
If $MoreHeader <> "" And $MoreHeader <> -1 And $MoreHeader <> Default Then _WinHttpAddRequestHeaders($hRequest,$MoreHeader)
; ==== 发送数据 ====
_WinHttpSendRequest($hRequest)
If @error Then Return SetError(4,0,_GetLastError_Winhttp())
; ==== 发送POST数据 ====
If $Context <> -1 And $Context <> Default And $Context <> "" Then
Switch $BinaryMode
Case 0
_WinHttpWriteData($hRequest,$Context)
Case 1
_WinHttpWriteData($hRequest,$Context,1)
EndSwitch
EndIf
; ==== 接受服务器回应 ====
_WinHttpReceiveResponse($hRequest)
; ==== 判断数据是否有效 ====
If Not _WinHttpQueryDataAvailable($hRequest) Then Return SetError(5,0,_GetLastError_Winhttp())
; ==== 自定义函数处理过程（此时开始调用外部函数） ====
If $FuncName <> "" And $FuncName <> -1 And $FuncName <> Default Then
If $ParamArray <> "" Then
Local $FuncArray
$FuncArray = "CallArgArray"
For $i = 1 To UBound($ParamArray)
$FuncArray[$i] = $ParamArray [$i -1]
If StringLeft($ParamArray [$i -1],1) = "$" Then $FuncArray[$i] = Eval(StringTrimLeft($FuncArray[$i],1))
Next
Call($FuncName,$FuncArray)
Else
Call($FuncName)
EndIf
If $Mode= 1 Then
_WinHttpCloseHandle($hRequest)
_WinHttpCloseHandle($hConnect)
If $SimpleMode = 1 Then _WinHttpCloseHandle($hOpen)
Return 1
EndIf
EndIf
; ==== 函数返回数据构建 ====
Local $iReturn = [-1,-1,-1,-1,-1,-1,-1,-1]
If $Mode = 1 Then $iReturn=_WinHttpQueryHeaders($hRequest) ;完整的Header信息
$iReturn=_WinHttpQueryHeaders($hRequest,$WINHTTP_QUERY_STATUS_CODE) ;服务器状态
$iReturn=_WinHttpQueryHeaders($hRequest,$WINHTTP_QUERY_CONTENT_LENGTH) ;文件长度
; ==== Cookies拆分 ====
$iReturn =_WinHttpQueryHeaders($hRequest,$WINHTTP_QUERY_SET_COOKIE,Default,0) ;Cookies with CRLF换行
If $iReturn <> "" Then
For $i = 1 To 20
Local $TempCookies = _WinHttpQueryHeaders($hRequest,$WINHTTP_QUERY_SET_COOKIE,Default,$i)
If $TempCookies <> "" Then
$iReturn &= @CRLF & $TempCookies
Else
$i = 21
$TempCookies = 0
EndIf
Next
EndIf
; ==== 完整的网址重定向构建 ====
$iReturn=_WinHttpQueryHeaders($hRequest,$WINHTTP_QUERY_LOCATION) ;重定向及其处理过程
If $iReturn <> "" Then
Local $RedictURL=_WinHttpCrackUrl($iReturn)
If @error Then
Local $RedictURL = $ConnectURL
$RedictURL = $iReturn
$RedictURL = ""
$iReturn = _WinHttpCreateUrl($RedictURL)
EndIf
$RedictURL = 0
EndIf
$iReturn=_WinHttpQueryHeaders($hRequest,$WINHTTP_QUERY_CONTENT_TYPE) ;编码类型
; ==== 返回-1替代空数据 ====
For $i = 1 To 5
If $iReturn[$i] = "" Then $iReturn[$i] = -1
Next
; ==== 读取返回正文数据 ====
If $Mode = 1 Then
Switch $iReturn
Case -1,""
$rDate =Binary("")
While 1
$rDate &= _WinHttpReadData($hRequest,2,1024 * 16)
If @error Then ExitLoop
Sleep(10)
WEnd
$iReturn = Binary($rDate)
Case Else
$iReturn = _WinHttpReadData($hRequest,2,$iReturn)
EndSwitch
; ==== 转回字符串 ====
If $Mode = 0 Then
Local $StringText = ['text/html','text/xml','application/xhtml+xml','text/plain'],$FindText = 0,$StringType = ["UTF-8"],$FindType = -1
For $i = 0 To UBound($StringText) -1
If StringInStr($iReturn,$StringText[$i]) Then
$FindText = 1
ExitLoop
EndIf
Next
If $FindText Then
For $i = 0 To 0
If StringInStr($iReturn,$StringType[$i]) Then
$FindType = $i
ExitLoop
EndIf
Next
Switch $FindType
Case 0
$iReturn = BinaryToString($iReturn,4)
Case Else
$iReturn = BinaryToString($iReturn)
EndSwitch
EndIf
EndIf
EndIf
; ==== 判断是否关闭句柄 ====
_WinHttpCloseHandle($hRequest)
Switch $Mode
Case 1
$iReturn = $hConnect
Case Else
_WinHttpCloseHandle($hConnect)
EndSwitch
If $SimpleMode = 1 Then _WinHttpCloseHandle($hOpen)
Return $iReturn
EndFunc
;-----LastError Meaning:
;~ http://msdn.microsoft.com/en-us/library/ms681381(VS.85).aspx
;~ http://msdn.microsoft.com/en-us/library/aa385465(v=VS.85).aspx
Func _GetLastError_Winhttp()
Local $lastError = DllCall ( "kernel32.dll", "dword", "GetLastError" )
Return $lastError
EndFunc
Func _NumberToBinary_Winhttp($iNumber)
Local $sBinString = "",$ModeNum = 11, $Numbers[$ModeNum]
Local $iUnsignedNumber=BitAND($iNumber,0x7FFFFFFF)
Do
$sBinString = BitAND($iUnsignedNumber, 1) & $sBinString
$iUnsignedNumber = BitShift($iUnsignedNumber, 1)
Until Not $iUnsignedNumber
$sBinString = StringRight("0000000000000000" &$sBinString,$ModeNum)
For $i = 0 To $ModeNum -1
$Numbers[$i] = StringMid($sBinString,$ModeNum - $i,1)
Next
Return $Numbers
EndFunc ;==>_NumberToBinary
Func _WinHttpProxyInfoCreate($dwAccessType, $sProxy, $sProxyBypass)
Local $tWINHTTP_PROXY_INFO = ; wchar proxybypasschars[' & StringLen($sProxyBypass)+1 & ']')]
DllStructSetData($tWINHTTP_PROXY_INFO, "dwAccessType", $dwAccessType)
If StringLen($sProxy) Then DllStructSetData($tWINHTTP_PROXY_INFO, "lpszProxy", DllStructGetPtr($tWINHTTP_PROXY_INFO, 'proxychars'))
If StringLen($sProxyByPass) Then DllStructSetData($tWINHTTP_PROXY_INFO, "lpszProxyBypass", DllStructGetPtr($tWINHTTP_PROXY_INFO, 'proxybypasschars'))
DllStructSetData($tWINHTTP_PROXY_INFO, "proxychars", $sProxy)
DllStructSetData($tWINHTTP_PROXY_INFO, "proxybypasschars", $sProxyBypass)
Return $tWINHTTP_PROXY_INFO
EndFunc 本帖最后由 heavenm 于 2016-4-21 03:07 编辑
发现问题鸟！是
BinaryToString 转换的时候有字符限制好像
太长了 本帖最后由 heavenm 于 2016-4-21 14:15 编辑
Local $_STR
$iReturn = StringTrimLeft($iReturn, 2)
Do
$_STR = BinaryToString('0x'&StringLeft($iReturn, 1024*20),4)
FileWriteLine ( @ScriptDir&"\1.txt", $_STR )
$iReturn = StringTrimLeft($iReturn, 1024*20)
Until StringIsSpace($iReturn)
$iReturn = $_STR
写入文本就不会丢失！是神马问题 我靠 终于发现终极问题了，竟然是一个变量储存的字符串有数量限制！！！！ 我靠 终于发现终极问题了，竟然是一个变量储存的字符串有数量限制！！！！
heavenm 发表于 2016-4-21 14:08 http://www.autoitx.com/images/common/back.gif
Au3限制： 二进制数据的最大字节或字符串的最大长度 2,147,483,647这么大应该足够用了吧 本帖最后由 heavenm 于 2016-4-21 14:19 编辑
Au3限制： 二进制数据的最大字节或字符串的最大长度 2,147,483,647这么大应该足够用了吧
afan 发表于 2016-4-21 14:15 http://www.autoitx.com/images/common/back.gif
奇怪转换成字符串的时候就会丢失
连接起来就会丢失
Local $_STR
$iReturn = StringTrimLeft($iReturn, 2)
Do
$_STR &= BinaryToString('0x'&StringLeft($iReturn, 1024*20),4)
$iReturn = StringTrimLeft($iReturn, 1024*20)
Until StringIsSpace($iReturn)
$iReturn = $_STR
写入文本不会丢失
Local $_STR
$iReturn = StringTrimLeft($iReturn, 2)
Do
$_STR = BinaryToString('0x'&StringLeft($iReturn, 1024*20),4)
FileWriteLine ( @ScriptDir&"\1.txt", $_STR )
$iReturn = StringTrimLeft($iReturn, 1024*20)
Until StringIsSpace($iReturn)
$iReturn = $_STR 本帖最后由 heavenm 于 2016-4-21 14:26 编辑
= =#不知道是不是剪切板问题，直接写入文本也是可以的 奇怪转换成字符串的时候就会丢失
连接起来就会丢失
...
heavenm 发表于 2016-4-21 14:17 http://www.autoitx.com/images/common/back.gif
估计你是输出限制 估计你是输出限制
afan 发表于 2016-4-21 14:28 http://www.autoitx.com/images/common/back.gif
终于知道为什么了！研究了一天
_ClipBoard_SetData($_TempHtml)输出文字会丢失
_ClipBoard_SetData($_TempHtml,$CF_UNICODETEXT)这样子就OK了 谢谢组织，谢谢AFAN版主
页:
[1]