mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 16:22:17 +00:00
1.0.0.2
Add: downloading certain types of media, name files by date Fix: Stop button is not activated after download start
This commit is contained in:
Andy
@@ -92,7 +92,7 @@ Namespace API.Reddit
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
Dim PostID$ = String.Empty
|
||||
Dim PostDate$
|
||||
Dim PostDate$, PostTitle$
|
||||
Dim n As EContainer, nn As EContainer, s As EContainer
|
||||
Dim NewPostDetected As Boolean = False
|
||||
Dim ExistsDetected As Boolean = False
|
||||
@@ -107,7 +107,7 @@ Namespace API.Reddit
|
||||
ThrowAny(Token)
|
||||
Dim r$ = GetSiteResponse(URL)
|
||||
If Not r.IsEmptyString Then
|
||||
Using w As EContainer = JsonDocument.Parse(r)
|
||||
Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
|
||||
If w.Count > 0 Then
|
||||
n = w.GetNode(JsonNodesJson)
|
||||
If Not n Is Nothing AndAlso n.Count > 0 Then
|
||||
@@ -124,29 +124,32 @@ Namespace API.Reddit
|
||||
ExistsDetected = True
|
||||
Continue For
|
||||
End If
|
||||
PostTitle = nn.Value("title")
|
||||
|
||||
If CheckNode(nn) Then
|
||||
_ItemsBefore = _TempMediaList.Count
|
||||
added = True
|
||||
s = nn.ItemF({"source", "url"})
|
||||
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
Else
|
||||
s = nn.ItemF({"media"}).XmlIfNothing
|
||||
__ItemType = s("type").XmlIfNothingValue
|
||||
Select Case __ItemType
|
||||
Case "gallery" : DownloadGallery(s, PostID, PostDate) : _TotalPostsDownloaded += 1
|
||||
Case "gallery" : If Not DownloadGallery(s, PostID, PostDate,,, PostTitle) Then added = False
|
||||
Case "image", "gifvideo"
|
||||
If s.Contains("content") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
|
||||
PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
Else
|
||||
added = False
|
||||
End If
|
||||
Case "video"
|
||||
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
|
||||
PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
Else
|
||||
added = False
|
||||
End If
|
||||
Case Else : added = False
|
||||
End Select
|
||||
@@ -164,8 +167,7 @@ Namespace API.Reddit
|
||||
End Select
|
||||
End With
|
||||
If Not tmpType = UTypes.Undefined Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
|
||||
_TotalPostsDownloaded += 1
|
||||
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
@@ -267,25 +269,30 @@ Namespace API.Reddit
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download Base Functions"
|
||||
Private Sub DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False)
|
||||
Private Function DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False,
|
||||
Optional ByVal Title As String = Nothing) As Boolean
|
||||
Try
|
||||
Dim added As Boolean = False
|
||||
Dim cn$ = IIf(IsChannel, "media_metadata", "mediaMetadata")
|
||||
If Not w Is Nothing AndAlso w(cn).XmlIfNothing.Count > 0 Then
|
||||
Dim t As EContainer
|
||||
For Each n As EContainer In w(cn)
|
||||
t = n.ItemF({"s", "u"})
|
||||
If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel), LNC)
|
||||
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel, Title), LNC)
|
||||
added = True
|
||||
If FirstOnly Then Exit For
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Return added
|
||||
Catch ex As Exception
|
||||
LogError(ex, "gallery parsing error")
|
||||
HasError = True
|
||||
Return False
|
||||
End Try
|
||||
End Sub
|
||||
End Function
|
||||
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
|
||||
Try
|
||||
ThrowAny(Token)
|
||||
@@ -333,13 +340,15 @@ Namespace API.Reddit
|
||||
#End Region
|
||||
#Region "Structure creator"
|
||||
Protected Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False) As UserMedia
|
||||
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False,
|
||||
Optional ByVal Title As String = Nothing) As UserMedia
|
||||
If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing
|
||||
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
|
||||
Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
|
||||
If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing
|
||||
If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
|
||||
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
|
||||
If Not Title.IsEmptyString Then m.Post.Title = Title
|
||||
Return m
|
||||
End Function
|
||||
Private Function TryFile(ByVal URL As String) As Boolean
|
||||
@@ -436,7 +445,11 @@ Namespace API.Reddit
|
||||
Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
|
||||
Case UTypes.Video, UTypes.m3u8 : DownloadedVideos += 1 : _CountVideo += 1
|
||||
End Select
|
||||
v.File = f
|
||||
If Not IsChannel Or Not SaveToCache Then
|
||||
v.File = ChangeFileNameByProvider(f, v)
|
||||
Else
|
||||
v.File = f
|
||||
End If
|
||||
v.Post.CachedFile = f
|
||||
v.State = UStates.Downloaded
|
||||
dCount += 1
|
||||
|
||||
Reference in New Issue
Block a user