mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-14 15:52:18 +00:00
2023.6.8.0
YT.Music: append artist name to music playlist output path YT.MediaItem: fixed opening paths to downloaded playlists and channels YT.YouTubeMediaContainerBase: save thumbnail path for playlist and channel UserDataBase: remove old line of code API.Twitter: fixed profile not fully downloaded SiteEditorForm: corrected form size for small monitors
This commit is contained in:
12
Changelog.md
12
Changelog.md
@@ -1,3 +1,15 @@
|
|||||||
|
# 2023.6.8.0
|
||||||
|
|
||||||
|
*2023-06-08*
|
||||||
|
|
||||||
|
- Added
|
||||||
|
- YouTube: append artist name to music playlist output path
|
||||||
|
- YouTube: save thumbnail path for playlist and channel
|
||||||
|
- Fixed
|
||||||
|
- YouTube: opening paths to downloaded playlists and channels
|
||||||
|
- Twitter: profile not fully downloaded
|
||||||
|
- Corrected form size for small monitors (Issue #136)
|
||||||
|
|
||||||
# 2023.6.5.0
|
# 2023.6.5.0
|
||||||
|
|
||||||
*2023-06-05*
|
*2023-06-05*
|
||||||
|
|||||||
@@ -81,6 +81,14 @@ Namespace API.YouTube.Controls
|
|||||||
|
|
||||||
If Not .UserTitle.IsEmptyString Then
|
If Not .UserTitle.IsEmptyString Then
|
||||||
Text = .UserTitle
|
Text = .UserTitle
|
||||||
|
If .ObjectType = Base.YouTubeMediaType.PlayList Then
|
||||||
|
If Not .PlaylistTitle.IsEmptyString AndAlso Not .PlaylistTitle = .UserTitle Then
|
||||||
|
Text &= $" - { .PlaylistTitle}"
|
||||||
|
ElseIf Not .Title.IsEmptyString AndAlso Not .Title = .UserTitle Then
|
||||||
|
Text &= $" - { .Title}"
|
||||||
|
End If
|
||||||
|
End If
|
||||||
|
If Not TXT_OUTPUT_PATH.IsEmptyString AndAlso Not TXT_OUTPUT_PATH.Text.Contains(.UserTitle) Then TXT_OUTPUT_PATH.Text = $"{TXT_OUTPUT_PATH.Text.TrimEnd("\")}\{ .UserTitle}\"
|
||||||
ElseIf Not .PlaylistTitle.IsEmptyString Then
|
ElseIf Not .PlaylistTitle.IsEmptyString Then
|
||||||
Text = .PlaylistTitle
|
Text = .PlaylistTitle
|
||||||
End If
|
End If
|
||||||
|
|||||||
@@ -115,31 +115,36 @@ Namespace DownloadObjects.STDownloader
|
|||||||
Me.New
|
Me.New
|
||||||
Const d$ = " " & ChrW(183) & " "
|
Const d$ = " " & ChrW(183) & " "
|
||||||
MyContainer = Container
|
MyContainer = Container
|
||||||
MyContainer.Progress = MyProgress
|
With MyContainer
|
||||||
If MyContainer.HasElements Then FileOption = SFO.Path Else FileOption = SFO.File
|
.Progress = MyProgress
|
||||||
If Not MyContainer.SiteKey = YouTubeSiteKey Then
|
If .HasElements Then FileOption = SFO.Path Else FileOption = SFO.File
|
||||||
|
If .DownloadState = Plugin.UserMediaStates.Downloaded AndAlso
|
||||||
|
(.ObjectType = Base.YouTubeMediaType.Channel Or .ObjectType = Base.YouTubeMediaType.PlayList) AndAlso FileOption = SFO.File AndAlso
|
||||||
|
Not .File.Exists AndAlso .File.Exists(SFO.Path, False) Then FileOption = SFO.Path
|
||||||
|
If Not .SiteKey = YouTubeSiteKey Then
|
||||||
BTT_DOWN_AGAIN.Visible = False
|
BTT_DOWN_AGAIN.Visible = False
|
||||||
SEP_DOWN_AGAIN.Visible = False
|
SEP_DOWN_AGAIN.Visible = False
|
||||||
End If
|
End If
|
||||||
|
|
||||||
ICON_SITE.Image = MyContainer.SiteIcon
|
ICON_SITE.Image = .SiteIcon
|
||||||
LBL_TIME.Text = AConvert(Of String)(Container.Duration, TimeToStringProvider, String.Empty)
|
LBL_TIME.Text = AConvert(Of String)(.Duration, TimeToStringProvider, String.Empty)
|
||||||
LBL_TITLE.Text = Container.ToString(True)
|
LBL_TITLE.Text = .ToString(True)
|
||||||
If Not Container.SiteKey = YouTubeSiteKey And Container.ContentType = Plugin.UserMediaTypes.Picture Then
|
If Not .SiteKey = YouTubeSiteKey And .ContentType = Plugin.UserMediaTypes.Picture Then
|
||||||
LBL_INFO.Text = Container.File.Extension.StringToUpper
|
LBL_INFO.Text = .File.Extension.StringToUpper
|
||||||
ElseIf Not Container.IsMusic Then
|
ElseIf Not .IsMusic Then
|
||||||
If Container.Height > 0 Then
|
If .Height > 0 Then
|
||||||
LBL_INFO.Text = $"{Container.File.Extension.StringToUpper}{d}{Container.Height}p"
|
LBL_INFO.Text = $"{ .File.Extension.StringToUpper}{d}{ .Height}p"
|
||||||
Else
|
Else
|
||||||
LBL_INFO.Text = Container.File.Extension.StringToUpper
|
LBL_INFO.Text = .File.Extension.StringToUpper
|
||||||
End If
|
End If
|
||||||
Else
|
Else
|
||||||
If Container.Bitrate > 0 Then
|
If .Bitrate > 0 Then
|
||||||
LBL_INFO.Text = $"{Container.File.Extension.StringToUpper}{d}{Container.Bitrate}k"
|
LBL_INFO.Text = $"{ .File.Extension.StringToUpper}{d}{ .Bitrate}k"
|
||||||
Else
|
Else
|
||||||
LBL_INFO.Text = Container.File.Extension.StringToUpper
|
LBL_INFO.Text = .File.Extension.StringToUpper
|
||||||
End If
|
End If
|
||||||
End If
|
End If
|
||||||
|
End With
|
||||||
UpdateMediaIcon()
|
UpdateMediaIcon()
|
||||||
End Sub
|
End Sub
|
||||||
#End Region
|
#End Region
|
||||||
|
|||||||
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
|
|||||||
' by using the '*' as shown below:
|
' by using the '*' as shown below:
|
||||||
' <Assembly: AssemblyVersion("1.0.*")>
|
' <Assembly: AssemblyVersion("1.0.*")>
|
||||||
|
|
||||||
<Assembly: AssemblyVersion("2023.6.5.0")>
|
<Assembly: AssemblyVersion("2023.6.8.0")>
|
||||||
<Assembly: AssemblyFileVersion("2023.6.5.0")>
|
<Assembly: AssemblyFileVersion("2023.6.8.0")>
|
||||||
<Assembly: NeutralResourcesLanguage("en")>
|
<Assembly: NeutralResourcesLanguage("en")>
|
||||||
|
|||||||
@@ -1040,6 +1040,14 @@ Namespace API.YouTube.Objects
|
|||||||
End Sub
|
End Sub
|
||||||
#End Region
|
#End Region
|
||||||
#Region "Save"
|
#Region "Save"
|
||||||
|
Private Function GetThumbnails() As IEnumerable(Of SFile)
|
||||||
|
If HasElements Then
|
||||||
|
Return ListAddList(Of SFile)(New List(Of SFile)({ThumbnailFile}),
|
||||||
|
Elements.SelectMany(Function(ee As YouTubeMediaContainerBase) ee.GetThumbnails))
|
||||||
|
Else
|
||||||
|
Return {ThumbnailFile}
|
||||||
|
End If
|
||||||
|
End Function
|
||||||
Public Overridable Sub Save() Implements IDownloadableMedia.Save
|
Public Overridable Sub Save() Implements IDownloadableMedia.Save
|
||||||
Try
|
Try
|
||||||
Dim fSettings As SFile = FileSettings
|
Dim fSettings As SFile = FileSettings
|
||||||
@@ -1068,6 +1076,11 @@ Namespace API.YouTube.Objects
|
|||||||
Else
|
Else
|
||||||
If CachePath.Exists(SFO.Path, False) Then CachePath.Delete(SFO.Path, SFODelete.DeletePermanently, EDP.None)
|
If CachePath.Exists(SFO.Path, False) Then CachePath.Delete(SFO.Path, SFODelete.DeletePermanently, EDP.None)
|
||||||
CachePath = Nothing
|
CachePath = Nothing
|
||||||
|
If ThumbnailFile.IsEmptyString And HasElements Then
|
||||||
|
With ListAddList(Nothing, GetThumbnails, LAP.NotContainsOnly).ListWithRemove(Function(tf) tf.IsEmptyString)
|
||||||
|
If .ListExists Then _ThumbnailFile = .FirstOrDefault(Function(tf) tf.Exists)
|
||||||
|
End With
|
||||||
|
End If
|
||||||
End If
|
End If
|
||||||
|
|
||||||
Using x As New XmlFile With {.AllowSameNames = True}
|
Using x As New XmlFile With {.AllowSameNames = True}
|
||||||
|
|||||||
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
|
|||||||
' by using the '*' as shown below:
|
' by using the '*' as shown below:
|
||||||
' <Assembly: AssemblyVersion("1.0.*")>
|
' <Assembly: AssemblyVersion("1.0.*")>
|
||||||
|
|
||||||
<Assembly: AssemblyVersion("2023.6.5.0")>
|
<Assembly: AssemblyVersion("2023.6.8.0")>
|
||||||
<Assembly: AssemblyFileVersion("2023.6.5.0")>
|
<Assembly: AssemblyFileVersion("2023.6.8.0")>
|
||||||
<Assembly: NeutralResourcesLanguage("en")>
|
<Assembly: NeutralResourcesLanguage("en")>
|
||||||
|
|||||||
@@ -1038,7 +1038,6 @@ BlockNullPicture:
|
|||||||
_ContentList.Clear()
|
_ContentList.Clear()
|
||||||
CreatedByChannel = False
|
CreatedByChannel = False
|
||||||
End If
|
End If
|
||||||
If Not UserExists Then ReadyForDownload = False
|
|
||||||
UpdateUserInformation()
|
UpdateUserInformation()
|
||||||
If _CollectionButtonsExists AndAlso _EnvirChanged Then UpdateButtonsColor()
|
If _CollectionButtonsExists AndAlso _EnvirChanged Then UpdateButtonsColor()
|
||||||
ElseIf _ForceSaveUserInfo Then
|
ElseIf _ForceSaveUserInfo Then
|
||||||
|
|||||||
@@ -97,22 +97,73 @@ Namespace API.Twitter
|
|||||||
Dim URL$ = String.Empty
|
Dim URL$ = String.Empty
|
||||||
Dim tCache As CacheKeeper = Nothing
|
Dim tCache As CacheKeeper = Nothing
|
||||||
Try
|
Try
|
||||||
|
Const entry$ = "entry"
|
||||||
Dim PostID$ = String.Empty
|
Dim PostID$ = String.Empty
|
||||||
Dim PostDate$, tmpUserId$
|
Dim PostDate$, tmpUserId$
|
||||||
Dim j As EContainer
|
Dim i%
|
||||||
Dim nn As EContainer
|
Dim dirIndx% = -1
|
||||||
Dim NewPostDetected As Boolean = False
|
Dim timelineNode As Predicate(Of EContainer) = Function(ee) ee.Value("type").StringToLower = "timelineaddentries"
|
||||||
Dim ExistsDetected As Boolean = False
|
Dim pinNode As Predicate(Of EContainer) = Function(ee) ee.Value("type").StringToLower = "timelinepinentry"
|
||||||
|
Dim entriesNode As Predicate(Of EContainer) = Function(ee) ee.Name = "entries" Or ee.Name = entry
|
||||||
|
Dim sourceIdPredicate As Predicate(Of EContainer) = Function(ee) ee.Name = "source_user_id_str" Or ee.Name = "source_user_id"
|
||||||
|
Dim p As Predicate(Of EContainer)
|
||||||
|
Dim pIndx%
|
||||||
|
Dim isOneNode As Boolean, isPins As Boolean, ExistsDetected As Boolean, userInfoParsed As Boolean = False
|
||||||
|
Dim j As EContainer, rootNode As EContainer, tmpNode As EContainer, nn As EContainer = Nothing
|
||||||
|
|
||||||
|
Dim __parseContainer As Func(Of EContainer, Boolean) =
|
||||||
|
Function(ByVal ee As EContainer) As Boolean
|
||||||
|
If dirIndx <= 1 Then
|
||||||
|
nn = ee({"content", "itemContent", "tweet_results", "result", "legacy"})
|
||||||
|
Else
|
||||||
|
nn = ee
|
||||||
|
End If
|
||||||
|
|
||||||
|
If Not nn.ListExists Then nn = ee({"content", "itemContent", "tweet_results", "result", "tweet", "legacy"})
|
||||||
|
If nn.ListExists Then
|
||||||
|
PostID = nn.Value("id_str").IfNullOrEmpty(nn.Value("id"))
|
||||||
|
|
||||||
|
'Date Pattern:
|
||||||
|
'Sat Jan 01 01:10:15 +0000 2000
|
||||||
|
If nn.Contains("created_at") Then PostDate = nn("created_at").Value Else PostDate = String.Empty
|
||||||
|
Select Case CheckDatesLimit(PostDate, Declarations.DateProvider)
|
||||||
|
Case DateResult.Skip, DateResult.Exit : Return False
|
||||||
|
End Select
|
||||||
|
|
||||||
|
If Not _TempPostsList.Contains(PostID) Then
|
||||||
|
_TempPostsList.Add(PostID)
|
||||||
|
ElseIf isPins Then
|
||||||
|
Return False
|
||||||
|
Else
|
||||||
|
ExistsDetected = True
|
||||||
|
Return False
|
||||||
|
End If
|
||||||
|
|
||||||
|
tmpUserId = nn({"retweeted_status_result", "result", "legacy", "user_id_str"}).XmlIfNothingValue
|
||||||
|
|
||||||
|
If tmpUserId.IsEmptyString Then tmpUserId = nn.ItemF({"extended_entities", "media", 0, sourceIdPredicate}).XmlIfNothingValue.
|
||||||
|
IfNullOrEmpty(nn.Value("user_id")).IfNullOrEmpty(nn.Value("user_id_str")).IfNullOrEmpty("/")
|
||||||
|
|
||||||
|
If Not ParseUserMediaOnly OrElse (Not ID.IsEmptyString AndAlso tmpUserId = ID) Then ObtainMedia(nn, PostID, PostDate)
|
||||||
|
End If
|
||||||
|
Return True
|
||||||
|
End Function
|
||||||
|
|
||||||
tCache = New CacheKeeper($"{DownloadContentDefault_GetRootDir()}\_tCache\")
|
tCache = New CacheKeeper($"{DownloadContentDefault_GetRootDir()}\_tCache\")
|
||||||
If tCache.RootDirectory.Exists(SFO.Path, False) Then tCache.RootDirectory.Delete(SFO.Path, SFODelete.DeletePermanently, EDP.ReturnValue)
|
If tCache.RootDirectory.Exists(SFO.Path, False) Then tCache.RootDirectory.Delete(SFO.Path, SFODelete.DeletePermanently, EDP.ReturnValue)
|
||||||
tCache.Validate()
|
tCache.Validate()
|
||||||
Dim f As SFile = GetTimelineFromGalleryDL(tCache.RootDirectory, Token)
|
|
||||||
If Not f.IsEmptyString Then
|
Dim dirs As List(Of SFile) = GetTimelineFromGalleryDL(tCache, Token)
|
||||||
|
If dirs.ListExists Then
|
||||||
|
For Each dir As SFile In dirs
|
||||||
|
dirIndx += 1
|
||||||
|
|
||||||
|
ExistsDetected = False
|
||||||
|
|
||||||
|
If Not dir.IsEmptyString Then
|
||||||
ThrowAny(Token)
|
ThrowAny(Token)
|
||||||
Dim timelineFiles As List(Of SFile) = SFile.GetFiles(f, "*.txt",, EDP.ReturnValue)
|
Dim timelineFiles As List(Of SFile) = SFile.GetFiles(dir, "*.txt",, EDP.ReturnValue)
|
||||||
If timelineFiles.ListExists Then
|
If timelineFiles.ListExists Then
|
||||||
Dim i%
|
|
||||||
ResetFileNameProvider(Math.Max(timelineFiles.Count.ToString.Length, 2))
|
ResetFileNameProvider(Math.Max(timelineFiles.Count.ToString.Length, 2))
|
||||||
'rename files
|
'rename files
|
||||||
For i = 0 To timelineFiles.Count - 1 : timelineFiles(i) = RenameGdlFile(timelineFiles(i), i) : Next
|
For i = 0 To timelineFiles.Count - 1 : timelineFiles(i) = RenameGdlFile(timelineFiles(i), i) : Next
|
||||||
@@ -121,6 +172,8 @@ Namespace API.Twitter
|
|||||||
j = JsonDocument.Parse(timelineFiles(i).GetText)
|
j = JsonDocument.Parse(timelineFiles(i).GetText)
|
||||||
If Not j Is Nothing Then
|
If Not j Is Nothing Then
|
||||||
If i = 0 Then
|
If i = 0 Then
|
||||||
|
If Not userInfoParsed Then
|
||||||
|
userInfoParsed = True
|
||||||
Dim resValue$ = j.Value({"data", "user", "result"}, "__typename").StringTrim.StringToLower
|
Dim resValue$ = j.Value({"data", "user", "result"}, "__typename").StringTrim.StringToLower
|
||||||
If resValue.IsEmptyString Then
|
If resValue.IsEmptyString Then
|
||||||
UserExists = False
|
UserExists = False
|
||||||
@@ -165,50 +218,56 @@ Namespace API.Twitter
|
|||||||
End If
|
End If
|
||||||
End With
|
End With
|
||||||
End If
|
End If
|
||||||
|
End If
|
||||||
Else
|
Else
|
||||||
With j({"globalObjects", "tweets"})
|
For pIndx = 0 To IIf(dirIndx < 2, 1, 0)
|
||||||
If .ListExists Then
|
Select Case dirIndx
|
||||||
ProgressPre.ChangeMax(.Count)
|
Case 0, 1
|
||||||
For Each nn In .Self
|
rootNode = j({"data", "user", "result", "timeline_v2", "timeline", "instructions"})
|
||||||
ProgressPre.Perform()
|
If rootNode.ListExists Then
|
||||||
If nn.Count > 0 Then
|
p = If(pIndx = 0, pinNode, timelineNode)
|
||||||
PostID = nn.Value("id")
|
isPins = pIndx = 0
|
||||||
|
rootNode = rootNode.Find(p, False)
|
||||||
'Date Pattern:
|
If rootNode.ListExists Then rootNode = rootNode.Find(entriesNode, False)
|
||||||
'Sat Jan 01 01:10:15 +0000 2000
|
End If
|
||||||
If nn.Contains("created_at") Then PostDate = nn("created_at").Value Else PostDate = String.Empty
|
Case Else
|
||||||
Select Case CheckDatesLimit(PostDate, Declarations.DateProvider)
|
isPins = False
|
||||||
Case DateResult.Skip : Continue For
|
rootNode = j({"globalObjects", "tweets"})
|
||||||
Case DateResult.Exit : Exit Sub
|
|
||||||
End Select
|
End Select
|
||||||
|
|
||||||
If Not _TempPostsList.Contains(PostID) Then
|
If rootNode.ListExists Then
|
||||||
NewPostDetected = True
|
With rootNode
|
||||||
_TempPostsList.Add(PostID)
|
isOneNode = dirIndx < 2 AndAlso .Name = entry
|
||||||
|
ProgressPre.ChangeMax(If(isOneNode, 1, .Count))
|
||||||
|
If isOneNode Then
|
||||||
|
ProgressPre.Perform()
|
||||||
|
If Not __parseContainer(.Self) Then Exit For
|
||||||
Else
|
Else
|
||||||
ExistsDetected = True
|
For Each tmpNode In .Self
|
||||||
Continue For
|
ProgressPre.Perform()
|
||||||
End If
|
If Not __parseContainer(tmpNode) Then Exit For
|
||||||
|
|
||||||
tmpUserId = nn.ItemF({"extended_entities", "media", 0, "source_user_id"}).
|
|
||||||
XmlIfNothingValue.IfNullOrEmpty(nn.Value("user_id")).IfNullOrEmpty("/")
|
|
||||||
|
|
||||||
If Not ParseUserMediaOnly OrElse (Not ID.IsEmptyString AndAlso tmpUserId = ID) Then _
|
|
||||||
ObtainMedia(nn, PostID, PostDate)
|
|
||||||
End If
|
|
||||||
Next
|
Next
|
||||||
End If
|
End If
|
||||||
End With
|
End With
|
||||||
End If
|
End If
|
||||||
|
Next
|
||||||
|
|
||||||
|
'TODO: Twitter: is this line needed?
|
||||||
|
If ExistsDetected And i = 1 Then Exit For Else ExistsDetected = False
|
||||||
|
End If
|
||||||
j.Dispose()
|
j.Dispose()
|
||||||
End If
|
End If
|
||||||
Next
|
Next
|
||||||
|
timelineFiles.Clear()
|
||||||
End If
|
End If
|
||||||
End If
|
End If
|
||||||
|
Next
|
||||||
|
End If
|
||||||
Catch ex As Exception
|
Catch ex As Exception
|
||||||
ProcessException(ex, Token, $"data downloading error [{URL}]")
|
ProcessException(ex, Token, $"data downloading error [{URL}]")
|
||||||
Finally
|
Finally
|
||||||
If Not tCache Is Nothing Then tCache.Dispose()
|
If Not tCache Is Nothing Then tCache.Dispose()
|
||||||
|
If _TempPostsList.Count > 0 Then _TempPostsList.Sort()
|
||||||
End Try
|
End Try
|
||||||
End Sub
|
End Sub
|
||||||
Private Sub DownloadData_SavedPosts(ByVal Token As CancellationToken)
|
Private Sub DownloadData_SavedPosts(ByVal Token As CancellationToken)
|
||||||
@@ -249,8 +308,10 @@ Namespace API.Twitter
|
|||||||
#End Region
|
#End Region
|
||||||
#Region "Obtain media"
|
#Region "Obtain media"
|
||||||
Private Sub ObtainMedia(ByVal e As EContainer, ByVal PostID As String, ByVal PostDate As String, Optional ByVal State As UStates = UStates.Unknown)
|
Private Sub ObtainMedia(ByVal e As EContainer, ByVal PostID As String, ByVal PostDate As String, Optional ByVal State As UStates = UStates.Unknown)
|
||||||
Dim s As EContainer = e.ItemF({"extended_entities", "media"})
|
Dim s As EContainer = e({"extended_entities", "media"})
|
||||||
If s Is Nothing OrElse s.Count = 0 Then s = e.ItemF({"retweeted_status", "extended_entities", "media"})
|
If If(s?.Count, 0) = 0 Then s = e({"retweeted_status", "extended_entities", "media"})
|
||||||
|
If If(s?.Count, 0) = 0 Then s = e({"retweeted_status_result", "result", "legacy", "extended_entities", "media"})
|
||||||
|
|
||||||
If If(s?.Count, 0) > 0 Then
|
If If(s?.Count, 0) > 0 Then
|
||||||
Dim mUrl$
|
Dim mUrl$
|
||||||
For Each m As EContainer In s
|
For Each m As EContainer In s
|
||||||
@@ -353,7 +414,7 @@ Namespace API.Twitter
|
|||||||
Friend Sub New(ByVal Dir As SFile, ByVal _Token As CancellationToken)
|
Friend Sub New(ByVal Dir As SFile, ByVal _Token As CancellationToken)
|
||||||
MyBase.New
|
MyBase.New
|
||||||
Commands.Clear()
|
Commands.Clear()
|
||||||
ChangeDirectory(Dir)
|
If Not Dir.IsEmptyString Then ChangeDirectory(Dir)
|
||||||
Token = _Token
|
Token = _Token
|
||||||
End Sub
|
End Sub
|
||||||
Protected Overrides Async Function Validate(ByVal Value As String) As Task
|
Protected Overrides Async Function Validate(ByVal Value As String) As Task
|
||||||
@@ -384,7 +445,7 @@ Namespace API.Twitter
|
|||||||
End If
|
End If
|
||||||
command &= URL
|
command &= URL
|
||||||
'#If DEBUG Then
|
'#If DEBUG Then
|
||||||
' Debug.WriteLine(command)
|
'Debug.WriteLine(command)
|
||||||
'#End If
|
'#End If
|
||||||
batch.Execute(command)
|
batch.Execute(command)
|
||||||
End Using
|
End Using
|
||||||
@@ -395,25 +456,40 @@ Namespace API.Twitter
|
|||||||
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetDataFromGalleryDL({command})")
|
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetDataFromGalleryDL({command})")
|
||||||
End Try
|
End Try
|
||||||
End Function
|
End Function
|
||||||
Private Function GetTimelineFromGalleryDL(ByVal Cache As CacheKeeper, ByVal Token As CancellationToken) As SFile
|
Private Function GetTimelineFromGalleryDL(ByVal Cache As CacheKeeper, ByVal Token As CancellationToken) As List(Of SFile)
|
||||||
Dim command$ = String.Empty
|
Dim command$ = String.Empty
|
||||||
Try
|
Try
|
||||||
Dim conf As SFile = $"{Cache.NewPath.PathWithSeparator}TwitterGdlConfig.conf"
|
Dim conf As SFile = $"{Cache.NewPath.PathWithSeparator}TwitterGdlConfig.conf"
|
||||||
Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") &
|
Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") &
|
||||||
""",""cookies-update"": false,""twitter"":{""cards"": false,""conversations"": false,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}"
|
""",""cookies-update"": false,""twitter"":{""cards"": false,""conversations"": true,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}"
|
||||||
If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf)
|
If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf)
|
||||||
If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf)
|
If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf)
|
||||||
|
|
||||||
|
Dim outList As New List(Of SFile)
|
||||||
|
Dim rootDir As CacheKeeper = Cache.NewInstance
|
||||||
|
Dim dir As SFile
|
||||||
|
|
||||||
|
Using tgdl As New TwitterGDL(Nothing, Token) With {.TempPostsList = _TempPostsList, .AutoClear = True, .AutoReset = True}
|
||||||
|
For i As Byte = 0 To 2
|
||||||
|
dir = rootDir.NewPath
|
||||||
|
dir.Exists(SFO.Path, True, EDP.ThrowException)
|
||||||
|
outList.Add(dir)
|
||||||
|
tgdl.ChangeDirectory(dir)
|
||||||
command = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --config ""{conf}"" --write-pages "
|
command = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --config ""{conf}"" --write-pages "
|
||||||
command &= GdlGetIdFilterString()
|
command &= GdlGetIdFilterString()
|
||||||
command &= $"https://twitter.com/search?q=from:{Name}+include:nativeretweets"
|
Select Case i
|
||||||
Dim dir As SFile = Cache.NewPath
|
Case 0 : command &= $"https://twitter.com/{Name}/media"
|
||||||
dir.Exists(SFO.Path, True, EDP.ThrowException)
|
Case 1 : command &= $"https://twitter.com/{Name}"
|
||||||
|
Case 2 : command &= $"https://twitter.com/search?q=from:{Name}+include:nativeretweets"
|
||||||
|
End Select
|
||||||
'#If DEBUG Then
|
'#If DEBUG Then
|
||||||
' Debug.WriteLine(command)
|
'Debug.WriteLine(command)
|
||||||
'#End If
|
'#End If
|
||||||
Using tgdl As New TwitterGDL(dir, Token) With {.TempPostsList = _TempPostsList} : tgdl.Execute(command) : End Using
|
tgdl.Execute(command)
|
||||||
Return dir
|
Next
|
||||||
|
End Using
|
||||||
|
|
||||||
|
Return outList
|
||||||
Catch ex As Exception
|
Catch ex As Exception
|
||||||
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetTimelineFromGalleryDL({command})")
|
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetTimelineFromGalleryDL({command})")
|
||||||
End Try
|
End Try
|
||||||
|
|||||||
6
SCrawler/Editors/SiteEditorForm.Designer.vb
generated
6
SCrawler/Editors/SiteEditorForm.Designer.vb
generated
@@ -54,7 +54,7 @@ Namespace Editors
|
|||||||
'CONTAINER_MAIN.ContentPanel
|
'CONTAINER_MAIN.ContentPanel
|
||||||
'
|
'
|
||||||
CONTAINER_MAIN.ContentPanel.Controls.Add(Me.TP_MAIN)
|
CONTAINER_MAIN.ContentPanel.Controls.Add(Me.TP_MAIN)
|
||||||
CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(544, 218)
|
CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(544, 243)
|
||||||
CONTAINER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
|
CONTAINER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
|
||||||
CONTAINER_MAIN.LeftToolStripPanelVisible = False
|
CONTAINER_MAIN.LeftToolStripPanelVisible = False
|
||||||
CONTAINER_MAIN.Location = New System.Drawing.Point(0, 0)
|
CONTAINER_MAIN.Location = New System.Drawing.Point(0, 0)
|
||||||
@@ -84,7 +84,7 @@ Namespace Editors
|
|||||||
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||||
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||||
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||||
Me.TP_MAIN.Size = New System.Drawing.Size(544, 218)
|
Me.TP_MAIN.Size = New System.Drawing.Size(544, 243)
|
||||||
Me.TP_MAIN.TabIndex = 0
|
Me.TP_MAIN.TabIndex = 0
|
||||||
'
|
'
|
||||||
'TXT_PATH
|
'TXT_PATH
|
||||||
@@ -135,7 +135,7 @@ Namespace Editors
|
|||||||
Me.TP_SITE_PROPS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
Me.TP_SITE_PROPS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||||
Me.TP_SITE_PROPS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
Me.TP_SITE_PROPS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||||
Me.TP_SITE_PROPS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
Me.TP_SITE_PROPS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||||
Me.TP_SITE_PROPS.Size = New System.Drawing.Size(538, 78)
|
Me.TP_SITE_PROPS.Size = New System.Drawing.Size(538, 103)
|
||||||
Me.TP_SITE_PROPS.TabIndex = 5
|
Me.TP_SITE_PROPS.TabIndex = 5
|
||||||
'
|
'
|
||||||
'TXT_PATH_SAVED_POSTS
|
'TXT_PATH_SAVED_POSTS
|
||||||
|
|||||||
@@ -90,6 +90,11 @@ Namespace Editors
|
|||||||
End If
|
End If
|
||||||
|
|
||||||
If .PropList.Count > 0 Then
|
If .PropList.Count > 0 Then
|
||||||
|
With TP_SITE_PROPS
|
||||||
|
With .RowStyles : .RemoveAt(.Count - 1) : End With
|
||||||
|
.RowCount -= 1
|
||||||
|
End With
|
||||||
|
|
||||||
Dim laAdded As Boolean = False
|
Dim laAdded As Boolean = False
|
||||||
Dim loAdded As Boolean = False
|
Dim loAdded As Boolean = False
|
||||||
Dim pArr() As Boolean
|
Dim pArr() As Boolean
|
||||||
@@ -134,6 +139,19 @@ Namespace Editors
|
|||||||
CH_GET_USER_MEDIA_ONLY.Padding = New PaddingE(CH_GET_USER_MEDIA_ONLY.Padding) With {.Left = offset}
|
CH_GET_USER_MEDIA_ONLY.Padding = New PaddingE(CH_GET_USER_MEDIA_ONLY.Padding) With {.Left = offset}
|
||||||
If c > 0 Or h <> 0 Then
|
If c > 0 Or h <> 0 Then
|
||||||
Dim ss As New Size(Size.Width, Size.Height + h + c)
|
Dim ss As New Size(Size.Width, Size.Height + h + c)
|
||||||
|
Dim minScrh% = Screen.AllScreens.Min(Function(scr) scr.WorkingArea.Height)
|
||||||
|
|
||||||
|
If ss.Height >= minScrh - 20 Then
|
||||||
|
ss.Height = minScrh - 40
|
||||||
|
With TP_SITE_PROPS
|
||||||
|
.AutoScroll = True
|
||||||
|
Dim p As Padding = .Padding
|
||||||
|
p.Right = 3
|
||||||
|
.Padding = p
|
||||||
|
.PerformLayout()
|
||||||
|
End With
|
||||||
|
End If
|
||||||
|
|
||||||
MinimumSize = ss
|
MinimumSize = ss
|
||||||
Size = ss
|
Size = ss
|
||||||
MaximumSize = ss
|
MaximumSize = ss
|
||||||
|
|||||||
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
|
|||||||
' by using the '*' as shown below:
|
' by using the '*' as shown below:
|
||||||
' <Assembly: AssemblyVersion("1.0.*")>
|
' <Assembly: AssemblyVersion("1.0.*")>
|
||||||
|
|
||||||
<Assembly: AssemblyVersion("2023.6.5.0")>
|
<Assembly: AssemblyVersion("2023.6.8.0")>
|
||||||
<Assembly: AssemblyFileVersion("2023.6.5.0")>
|
<Assembly: AssemblyFileVersion("2023.6.8.0")>
|
||||||
<Assembly: NeutralResourcesLanguage("en")>
|
<Assembly: NeutralResourcesLanguage("en")>
|
||||||
|
|||||||
Reference in New Issue
Block a user