diff --git a/DataImporterTool/DataImporterTool.csproj b/DataImporterTool/DataImporterTool.csproj index 8e6af23..b9afde5 100644 --- a/DataImporterTool/DataImporterTool.csproj +++ b/DataImporterTool/DataImporterTool.csproj @@ -7,6 +7,8 @@ + + diff --git a/DataImporterTool/IMainFormView.cs b/DataImporterTool/IMainFormView.cs index f2307e1..d128a91 100644 --- a/DataImporterTool/IMainFormView.cs +++ b/DataImporterTool/IMainFormView.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; namespace DataImporterTool { @@ -10,6 +11,8 @@ namespace DataImporterTool string StatusInformation { get; set; } + public string SqlConnectionString { get; set; } + void ShowJsonFiles(string[] allJsonFiles); void ShowJsonAccountFiles(string[] accountJsonFiles); @@ -22,6 +25,8 @@ namespace DataImporterTool void SetProcessPercentage(int percentage); + void ShowSqlAccounts(Dictionary accounts); + event Action OpenJsonFolderClick; event Action AddFilesAsAccountsInfoList; @@ -29,5 +34,9 @@ namespace DataImporterTool event Action AddFilesAsTanksInfoList; event Action StartJsonConvert; + + event Action FetchSqlAccounts; + + event Action StartSqlImport; } } \ No newline at end of file diff --git a/DataImporterTool/Importer/AccountAndTanksContextRow.cs b/DataImporterTool/Importer/AccountAndTanksContextRow.cs new file mode 100644 index 0000000..cf838c0 --- /dev/null +++ b/DataImporterTool/Importer/AccountAndTanksContextRow.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using WotBlitzStatisticsPro.DataAccess.Model.Accounts; + +namespace DataImporterTool.Importer +{ + public class AccountAndTanksContextRow + { + public AccountInfo AccountInfo { get; set; } + public AccountInfoHistory AccountInfoHistory { get; set; } + public List Tanks { get; set; } + public Dictionary TanksHistory { get; set; } + } +} \ No newline at end of file diff --git a/DataImporterTool/Importer/FileImportProgress.cs b/DataImporterTool/Importer/FileImportProgress.cs new file mode 100644 index 0000000..ecb31d4 --- /dev/null +++ b/DataImporterTool/Importer/FileImportProgress.cs @@ -0,0 +1,12 @@ +namespace DataImporterTool.Importer +{ + public class FileImportProgress: ImportProgress + { + public int FilesConverted { get; set; } + + public string AccountFileConverted { get; set; } + + public string TankFileConverted { get; set; } + + } +} \ No newline at end of file diff --git a/DataImporterTool/Importer/ImportProgress.cs b/DataImporterTool/Importer/ImportProgress.cs index b18a2d7..3789dc1 100644 --- a/DataImporterTool/Importer/ImportProgress.cs +++ b/DataImporterTool/Importer/ImportProgress.cs @@ -4,14 +4,8 @@ namespace DataImporterTool.Importer { public class ImportProgress { - public int FilesConverted { get; set; } - public int Progress { get; set; } - public string AccountFileConverted { get; set; } - - public string TankFileConverted { get; set; } - public string AccountName { get; set; } public DateTime LastBattle { get; set; } diff --git a/DataImporterTool/Importer/JsonFilesImporter.cs b/DataImporterTool/Importer/JsonFilesImporter.cs index 5c4dd44..d6c3e3b 100644 --- a/DataImporterTool/Importer/JsonFilesImporter.cs +++ b/DataImporterTool/Importer/JsonFilesImporter.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; using AutoMapper; using Microsoft.Extensions.DependencyInjection; @@ -34,7 +33,7 @@ namespace DataImporterTool.Importer string folderPath, string[] accountsFiles, string[] tankFiles, - IProgress progress) + IProgress progress) { if (accountsFiles == null || tankFiles == null || accountsFiles.Length == 0 || accountsFiles.Length != tankFiles.Length) { @@ -49,7 +48,7 @@ namespace DataImporterTool.Importer Path.Combine(folderPath, accountsFiles[i]), Path.Combine(folderPath, tankFiles[i])); - var p = new ImportProgress + var p = new FileImportProgress { Progress = 100 * (i+1) / accountsFiles.Length, TankFileConverted = tankFiles[i], diff --git a/DataImporterTool/Importer/OldStatisticsDbContext.cs b/DataImporterTool/Importer/OldStatisticsDbContext.cs new file mode 100644 index 0000000..eea42c7 --- /dev/null +++ b/DataImporterTool/Importer/OldStatisticsDbContext.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore; +using WotBlitzStatisticsPro.DataAccess.Model.Accounts; + +namespace DataImporterTool.Importer +{ + public class OldStatisticsDbContext : DbContext + { + public OldStatisticsDbContext(DbContextOptions options) : base(options) + { + } + + public DbSet AccountInfos { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().HasKey(a => a.AccountId); + modelBuilder.Entity().Ignore(a => a.Realm); + modelBuilder.Entity().Ignore(a => a.UpdatedAt); + base.OnModelCreating(modelBuilder); + } + } +} \ No newline at end of file diff --git a/DataImporterTool/Importer/SqlImporter.cs b/DataImporterTool/Importer/SqlImporter.cs new file mode 100644 index 0000000..150c43e --- /dev/null +++ b/DataImporterTool/Importer/SqlImporter.cs @@ -0,0 +1,425 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Linq; +using System.Threading.Tasks; +using AutoMapper; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using WotBlitzStatisticsPro.Common; +using WotBlitzStatisticsPro.Common.Model; +using WotBlitzStatisticsPro.DataAccess; +using WotBlitzStatisticsPro.DataAccess.Model.Accounts; +using WotBlitzStatisticsPro.Logic; +using WotBlitzStatisticsPro.Logic.AccountInformationPipeline; +using WotBlitzStatisticsPro.Logic.AccountInformationPipeline.OperationContext; +using WotBlitzStatisticsPro.Logic.AccountInformationPipeline.Operations; +using WotBlitzStatisticsPro.Logic.Calculations; +using WotBlitzStatisticsPro.Logic.Pipeline; + +namespace DataImporterTool.Importer +{ + public class SqlImporter + { + private IOperationFactory _operationFactory = null; + private IMapper _mapper = null; + + public async Task?> FetchAllAccounts(string connectionString) + { + using (var context = await CreateDbContext(connectionString)) + { + var accounts = await context.AccountInfos.FromSqlRaw(FetchAllAccountsQuery).ToListAsync(); + if (accounts == null || accounts.Count == 0) + { + return null; + } + + return accounts.ToDictionary(a => a.AccountId, a => a.Nickname); + } + } + + public async Task StartImport( + string mongoConnectionString, + string sqlConnectionString, + long[] accountsToImport, + IProgress progress) + { + // Get summary history count - for percentage calculation + // Here I'm using simple query in the loop. But there is the way to pass an array as SQL parameter - using DataTable + int historyCount = 0; + using (var context = await CreateDbContext(sqlConnectionString)) + { + var connection = context.Database.GetDbConnection(); + foreach (long accountId in accountsToImport) + { + historyCount += await GetAccountHistoryCount(accountId, connection); + } + if (connection.State.Equals(ConnectionState.Open)) { await connection.CloseAsync(); } + } + + var allOldData = new AccountAndTanksContextRow[historyCount]; + await GetAllDataFromOldDatabase(sqlConnectionString, accountsToImport, progress, allOldData, historyCount); + + SetupOperations(mongoConnectionString); + + int historyCounter = historyCount; + for (int i = 0; i < historyCount; i++) + { + var saved = await SaveAccountAndTankInformation(allOldData[i]); + + var p = new ImportProgress + { + Progress = 100 * (historyCounter + 1) / (2 * historyCount), + AccountName = saved.Nickname, + LastBattle = saved.LastBattleTime, + BattlesCount = saved.Battles, + TanksCount = saved.Tanks?.Count ?? 0 + }; + progress.Report(p); + historyCounter++; + } + } + + private async Task SaveAccountAndTankInformation(AccountAndTanksContextRow accountAndTanksInfo) + { + var contextData = new AccountInformationPipelineContextData + { + AccountInfo = accountAndTanksInfo.AccountInfo, + AccountInfoHistory = accountAndTanksInfo.AccountInfoHistory, + Tanks = accountAndTanksInfo.Tanks, + TanksHistory = accountAndTanksInfo.TanksHistory + }; + + var context = new OperationContext(new AccountRequest(contextData.AccountInfo?.AccountId ?? -1, RealmType.Ru, RequestLanguage.En)); + context.AddOrReplace(contextData); + var pipeline = new Pipeline(_operationFactory); + + pipeline + .AddOperation() + .AddOperation() + .AddOperation() + ; + + var firstOperation = pipeline.Build(); + if (firstOperation != null) + { + await firstOperation + .Invoke(context, null) + .ConfigureAwait(false); + } + return contextData?.Response ?? new AccountInfoResponse(); + } + + private void SetupOperations( + string mongoConnectionString) + { + string databaseName = "WotBlitzStatisticsPro"; + + var settings = new MongoSettings { ConnectionString = mongoConnectionString, DatabaseName = databaseName }; + + var serviceProvider = new ServiceCollection() + .AddAutoMapper(typeof(WotBlitzStatisticsLogicInstaller)) + .AddLogging(configure => configure.AddConsole()) + .AddSingleton(settings) + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient(sp => + new ServiceProviderOperationFactory(sp)) + .BuildServiceProvider(); + + _operationFactory = serviceProvider.GetService(); + _mapper = serviceProvider.GetService(); + } + + + #region SQL Database stuff + private async Task GetAllDataFromOldDatabase(string sqlConnectionString, long[] accountsToImport, IProgress progress, + AccountAndTanksContextRow[] allOldData, int historyCount) + { + int historyCounter = 0; + + // In loop, get account info, history rows and tank history rows and collect to List + using (var context = await CreateDbContext(sqlConnectionString)) + { + var connection = context.Database.GetDbConnection(); + + foreach (long accountId in accountsToImport) + { + var accountInfo = await context.AccountInfos.FromSqlRaw( + SelectAccountInfoQuery, + new SqlParameter("AccountId", SqlDbType.Int) {Value = accountId}).FirstOrDefaultAsync(); + + var allHistory = await GetAccountInfoHistory(accountInfo.AccountId, connection); + foreach (var accountInfoHistory in allHistory) + { + allOldData[historyCounter] = new AccountAndTanksContextRow + { + AccountInfo = new AccountInfo + { + AccountId = accountId, + CreatedAt = accountInfo.CreatedAt, + LastBattleTime = accountInfoHistory.AccountInfoHistory.LastBattleTime, + Nickname = accountInfo.Nickname, + Realm = RealmType.Ru, + UpdatedAt = accountInfoHistory.UpdatedAt + }, + AccountInfoHistory = accountInfoHistory.AccountInfoHistory, + Tanks = new List(), + TanksHistory = new Dictionary() + }; + + var oldTanksListByUpdateDate = await GetTanksHistory(accountId, + allOldData[historyCounter].AccountInfo.LastBattleTime, connection); + + foreach (var tankInfo in oldTanksListByUpdateDate) + { + allOldData[historyCounter].Tanks.Add(new TankInfo(accountId, tankInfo.history.TankId) + { + LastBattleTime = tankInfo.history.LastBattleTime, + BattleLifeTimeInSeconds = tankInfo.BattleLifetime, + MarkOfMastery = tankInfo.Mastery + }); + allOldData[historyCounter].TanksHistory[tankInfo.history.TankId] = tankInfo.history; + } + + var p = new ImportProgress + { + Progress = 100 * (historyCounter + 1) / (2 * historyCount), + AccountName = accountInfo.Nickname, + LastBattle = accountInfoHistory.AccountInfoHistory.LastBattleTime.ToDateTime(), + BattlesCount = accountInfoHistory.AccountInfoHistory.Battles ?? 0, + TanksCount = oldTanksListByUpdateDate.Count + }; + progress.Report(p); + historyCounter++; + } + } + + if (connection.State.Equals(ConnectionState.Open)) + { + await connection.CloseAsync(); + } + } + } + + private async Task GetAccountHistoryCount(long accountId, DbConnection connection) + { + using (var command = connection.CreateCommand()) + { + command.CommandText = AccountHistoryCountQuery; + command.Parameters.Add(new SqlParameter("AccountId", SqlDbType.Int) { Value = accountId }); + if (connection.State.Equals(ConnectionState.Closed)) { await connection.OpenAsync(); } + + return (int)(await command.ExecuteScalarAsync() ?? 0); + } + } + + private async Task> GetAccountInfoHistory(long accountId, DbConnection connection) + { + var resultList = new List<(AccountInfoHistory AccountInfoHistory, int UpdatedAt)>(); + // As far as AccountInfoHistory has properties that can be set only via constructor, we use SQLDataReader here instead of DBSet + using (var command = connection.CreateCommand()) + { + command.CommandText = AccountInfoHistoryQuery; + command.Parameters.Add(new SqlParameter("AccountId", SqlDbType.Int) { Value = accountId }); + if (connection.State.Equals(ConnectionState.Closed)) { await connection.OpenAsync(); } + + using (var reader = await command.ExecuteReaderAsync()) + { + while (await reader.ReadAsync()) + { + var history = new AccountInfoHistory(reader.GetInt64(reader.GetOrdinal("AccountId")), + reader.GetInt32(reader.GetOrdinal("LastBattleTime"))) + { + Battles = reader.GetInt64(reader.GetOrdinal("Battles")), + CapturePoints = reader.GetInt64(reader.GetOrdinal("CapturePoints")), + DamageDealt = reader.GetInt64(reader.GetOrdinal("DamageDealt")), + DamageReceived = reader.GetInt64(reader.GetOrdinal("DamageReceived")), + DroppedCapturePoints = reader.GetInt64(reader.GetOrdinal("DroppedCapturePoints")), + Frags = reader.GetInt64(reader.GetOrdinal("Frags")), + Frags8P = reader.GetInt64(reader.GetOrdinal("Frags8P")), + Hits = reader.GetInt64(reader.GetOrdinal("Hits")), + Losses = reader.GetInt64(reader.GetOrdinal("Losses")), + MaxFrags = reader.GetInt64(reader.GetOrdinal("MaxFrags")), + MaxFragsTankId = reader.GetInt64(reader.GetOrdinal("MaxFragsTankId")), + MaxXp = reader.GetInt64(reader.GetOrdinal("MaxXp")), + MaxXpTankId = reader.GetInt64(reader.GetOrdinal("MaxXpTankId")), + Shots = reader.GetInt64(reader.GetOrdinal("Shots")), + Spotted = reader.GetInt64(reader.GetOrdinal("Spotted")), + SurvivedBattles = reader.GetInt64(reader.GetOrdinal("SurvivedBattles")), + WinAndSurvived = reader.GetInt64(reader.GetOrdinal("WinAndSurvived")), + Wins = reader.GetInt64(reader.GetOrdinal("Wins")), + Xp = reader.GetInt64(reader.GetOrdinal("Xp")), + AvgTier = reader.GetDouble(reader.GetOrdinal("AvgTier")), + Wn7 = reader.GetDouble(reader.GetOrdinal("Wn7")) + }; + resultList.Add((history, reader.GetInt32(reader.GetOrdinal("LastBattleTime")))); + } + } + } + + return resultList; + } + + private async Task> GetTanksHistory( + long accountId, + int lastBattleTime, + DbConnection connection) + { + var resultList = new List<(MarkOfMastery Mastery, int BattleLifetime, TankInfoHistory history)>(); + + using (var command = connection.CreateCommand()) + { + command.CommandText = TanksInfoHistoryQuery; + command.Parameters.Add(new SqlParameter("LastBattleTime", SqlDbType.DateTime) { Value = lastBattleTime.ToDateTime() }); + command.Parameters.Add(new SqlParameter("AccountId", SqlDbType.Int) { Value = accountId }); + if (connection.State.Equals(ConnectionState.Closed)) { await connection.OpenAsync(); } + + using (var reader = await command.ExecuteReaderAsync()) + { + while (await reader.ReadAsync()) + { + var tankHistory = new TankInfoHistory( + reader.GetInt64(reader.GetOrdinal("AccountId")), + reader.GetInt64(reader.GetOrdinal("TankId")), + reader.GetInt32(reader.GetOrdinal("LastBattleTime"))) + { + Battles = reader.GetInt64(reader.GetOrdinal("Battles")), + CapturePoints = reader.GetInt64(reader.GetOrdinal("CapturePoints")), + DamageDealt = reader.GetInt64(reader.GetOrdinal("DamageDealt")), + DamageReceived = reader.GetInt64(reader.GetOrdinal("DamageReceived")), + DroppedCapturePoints = reader.GetInt64(reader.GetOrdinal("DroppedCapturePoints")), + Frags = reader.GetInt64(reader.GetOrdinal("Frags")), + Frags8P = reader.GetInt64(reader.GetOrdinal("Frags8P")), + Hits = reader.GetInt64(reader.GetOrdinal("Hits")), + Losses = reader.GetInt64(reader.GetOrdinal("Losses")), + MaxFrags = reader.GetInt64(reader.GetOrdinal("MaxFrags")), + MaxXp = reader.GetInt64(reader.GetOrdinal("MaxXp")), + Shots = reader.GetInt64(reader.GetOrdinal("Shots")), + Spotted = reader.GetInt64(reader.GetOrdinal("Spotted")), + SurvivedBattles = reader.GetInt64(reader.GetOrdinal("SurvivedBattles")), + WinAndSurvived = reader.GetInt64(reader.GetOrdinal("WinAndSurvived")), + Wins = reader.GetInt64(reader.GetOrdinal("Wins")), + Xp = reader.GetInt64(reader.GetOrdinal("Xp")), + Wn7 = reader.GetDouble(reader.GetOrdinal("Wn7")) + }; + resultList.Add(( + (MarkOfMastery) reader.GetInt32(reader.GetOrdinal("MarkOfMastery")), + reader.GetInt32(reader.GetOrdinal("BattleLifeTimeInSeconds")), + tankHistory)); + } + } + + } + + return resultList; + } + + private async Task CreateDbContext(string connection) + { + var contextBuilder = new DbContextOptionsBuilder(); + contextBuilder.UseSqlServer(connection); + + var context = new OldStatisticsDbContext(contextBuilder.Options); + var connectionValid = await context.Database.CanConnectAsync(); + + if (!connectionValid) + { + await context.DisposeAsync(); + throw new ApplicationException("Connection string is not valid"); + } + + return context; + } + + private const string FetchAllAccountsQuery = @" +select AccountId AccountId, + wotb.to_unix_time(AccountCreatedAt) CreatedAt, + wotb.to_unix_time(LastBattleTime) LastBattleTime, + NickName Nickname +from wotb.AccountInfo +where AccountCreatedAt is not null and LastBattleTime is not null"; + + private const string AccountHistoryCountQuery = @" +select COUNT(*) +from wotb.AccountInfoStatistics +where AccountId = @AccountId"; + + private const string SelectAccountInfoQuery = @" +select AccountId AccountId, + wotb.to_unix_time(AccountCreatedAt) CreatedAt, + wotb.to_unix_time(LastBattleTime) LastBattleTime, + NickName Nickname +from wotb.AccountInfo +where AccountId = @AccountId"; + + private const string AccountInfoHistoryQuery = @" +select + s.AccountId, + wotb.to_unix_time(s.UpdatedAt) LastBattleTime, + s.Battles, + s.CapturePoints, + s.DamageDealt, + s.DamageReceived, + s.DroppedCapturePoints, + s.Frags, + s.Frags8P, + s.Hits, + s.Losses, + s.MaxFrags, + s.MaxFragsTankId, + s.MaxXp, + s.MaxXpTankId, + s.Shots, + s.Spotted, + s.SurvivedBattles, + s.WinAndSurvived, + s.Wins, + s.Xp, + s.AvgTier, + s.Wn7 +from wotb.AccountInfoStatistics s +where s.AccountId = @AccountId +order by s.UpdatedAt"; + + private const string TanksInfoHistoryQuery = @" +select + t.AccountId, + t.TankId, + wotb.to_unix_time(t.LastBattleTime) LastBattleTime, + t.BattleLifeTimeInSeconds, + t.MarkOfMastery, + t.Battles, + t.CapturePoints, + t.DamageDealt, + t.DamageReceived, + t.DroppedCapturePoints, + t.Frags, + t.Frags8P, + t.Hits, + t.Losses, + t.MaxFrags, + t.MaxXp, + t.Shots, + t.Spotted, + t.SurvivedBattles, + t.WinAndSurvived, + t.Wins, + t.Xp, + t.Wn7 +from wotb.AccountTankStatistics t + inner join (select MAX(LastBattleTime) MaxBattleTime, AccountId, TankId from wotb.AccountTankStatistics where LastBattleTime <= @LastBattleTime group by AccountId, TankId) q + on q.MaxBattleTime = t.LastBattleTime and q.AccountId = t.AccountId and q.TankId = t.TankId +where t.AccountId = @AccountId +order by t.LastBattleTime desc"; + + #endregion + } +} \ No newline at end of file diff --git a/DataImporterTool/MainForm.Designer.cs b/DataImporterTool/MainForm.Designer.cs index d09d2a6..9729039 100644 --- a/DataImporterTool/MainForm.Designer.cs +++ b/DataImporterTool/MainForm.Designer.cs @@ -46,6 +46,12 @@ namespace DataImporterTool this.btnOpenFolder = new System.Windows.Forms.Button(); this.txtJsonFolder = new System.Windows.Forms.TextBox(); this.tabPgSqlImport = new System.Windows.Forms.TabPage(); + this.btnImportSqlAccounts = new System.Windows.Forms.Button(); + this.lstSqlAccounts = new System.Windows.Forms.ListBox(); + this.label5 = new System.Windows.Forms.Label(); + this.btnFetchData = new System.Windows.Forms.Button(); + this.txtSqlConnectionString = new System.Windows.Forms.TextBox(); + this.label4 = new System.Windows.Forms.Label(); this.statusStrip = new System.Windows.Forms.StatusStrip(); this.statusImportLabel = new System.Windows.Forms.ToolStripStatusLabel(); this.statusImportProgressBar = new System.Windows.Forms.ToolStripProgressBar(); @@ -55,6 +61,7 @@ namespace DataImporterTool this.tabImportType.SuspendLayout(); this.tabPgFileImport.SuspendLayout(); this.grpFolder.SuspendLayout(); + this.tabPgSqlImport.SuspendLayout(); this.statusStrip.SuspendLayout(); this.SuspendLayout(); // @@ -233,6 +240,12 @@ namespace DataImporterTool // // tabPgSqlImport // + this.tabPgSqlImport.Controls.Add(this.btnImportSqlAccounts); + this.tabPgSqlImport.Controls.Add(this.lstSqlAccounts); + this.tabPgSqlImport.Controls.Add(this.label5); + this.tabPgSqlImport.Controls.Add(this.btnFetchData); + this.tabPgSqlImport.Controls.Add(this.txtSqlConnectionString); + this.tabPgSqlImport.Controls.Add(this.label4); this.tabPgSqlImport.Location = new System.Drawing.Point(4, 24); this.tabPgSqlImport.Name = "tabPgSqlImport"; this.tabPgSqlImport.Padding = new System.Windows.Forms.Padding(3); @@ -241,6 +254,62 @@ namespace DataImporterTool this.tabPgSqlImport.Text = "Import from Sql database"; this.tabPgSqlImport.UseVisualStyleBackColor = true; // + // btnImportSqlAccounts + // + this.btnImportSqlAccounts.Location = new System.Drawing.Point(691, 112); + this.btnImportSqlAccounts.Name = "btnImportSqlAccounts"; + this.btnImportSqlAccounts.Size = new System.Drawing.Size(126, 56); + this.btnImportSqlAccounts.TabIndex = 5; + this.btnImportSqlAccounts.Text = "Import selected accounts"; + this.btnImportSqlAccounts.UseVisualStyleBackColor = true; + // + // lstSqlAccounts + // + this.lstSqlAccounts.FormattingEnabled = true; + this.lstSqlAccounts.ItemHeight = 15; + this.lstSqlAccounts.Location = new System.Drawing.Point(10, 60); + this.lstSqlAccounts.Name = "lstSqlAccounts"; + this.lstSqlAccounts.SelectionMode = System.Windows.Forms.SelectionMode.MultiExtended; + this.lstSqlAccounts.Size = new System.Drawing.Size(674, 184); + this.lstSqlAccounts.TabIndex = 4; + // + // label5 + // + this.label5.AutoSize = true; + this.label5.Location = new System.Drawing.Point(10, 41); + this.label5.Name = "label5"; + this.label5.Size = new System.Drawing.Size(57, 15); + this.label5.TabIndex = 3; + this.label5.Text = "Accounts"; + // + // btnFetchData + // + this.btnFetchData.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right))); + this.btnFetchData.Location = new System.Drawing.Point(691, 6); + this.btnFetchData.Name = "btnFetchData"; + this.btnFetchData.Size = new System.Drawing.Size(126, 23); + this.btnFetchData.TabIndex = 2; + this.btnFetchData.Text = "Fetch Data"; + this.btnFetchData.UseVisualStyleBackColor = true; + // + // txtSqlConnectionString + // + this.txtSqlConnectionString.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) + | System.Windows.Forms.AnchorStyles.Right))); + this.txtSqlConnectionString.Location = new System.Drawing.Point(146, 7); + this.txtSqlConnectionString.Name = "txtSqlConnectionString"; + this.txtSqlConnectionString.Size = new System.Drawing.Size(538, 23); + this.txtSqlConnectionString.TabIndex = 1; + // + // label4 + // + this.label4.AutoSize = true; + this.label4.Location = new System.Drawing.Point(10, 10); + this.label4.Name = "label4"; + this.label4.Size = new System.Drawing.Size(130, 15); + this.label4.TabIndex = 0; + this.label4.Text = "SQL connection string: "; + // // statusStrip // this.statusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { @@ -271,6 +340,7 @@ namespace DataImporterTool this.txtIpmortLog.Multiline = true; this.txtIpmortLog.Name = "txtIpmortLog"; this.txtIpmortLog.ReadOnly = true; + this.txtIpmortLog.ScrollBars = System.Windows.Forms.ScrollBars.Vertical; this.txtIpmortLog.Size = new System.Drawing.Size(825, 170); this.txtIpmortLog.TabIndex = 3; // @@ -300,6 +370,8 @@ namespace DataImporterTool this.tabPgFileImport.PerformLayout(); this.grpFolder.ResumeLayout(false); this.grpFolder.PerformLayout(); + this.tabPgSqlImport.ResumeLayout(false); + this.tabPgSqlImport.PerformLayout(); this.statusStrip.ResumeLayout(false); this.statusStrip.PerformLayout(); this.ResumeLayout(false); @@ -331,6 +403,12 @@ namespace DataImporterTool private System.Windows.Forms.ListBox lstTanks; private System.Windows.Forms.ListBox lstAccounts; private System.Windows.Forms.Button btnStartImport; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.Button btnFetchData; + private System.Windows.Forms.TextBox txtSqlConnectionString; + private System.Windows.Forms.Label label5; + private System.Windows.Forms.Button btnImportSqlAccounts; + private System.Windows.Forms.ListBox lstSqlAccounts; } } diff --git a/DataImporterTool/MainForm.cs b/DataImporterTool/MainForm.cs index 95b8814..d0e03cb 100644 --- a/DataImporterTool/MainForm.cs +++ b/DataImporterTool/MainForm.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Windows.Forms; @@ -17,6 +18,11 @@ namespace DataImporterTool btnAddAsAccounts.Click += BtnAddAsAccounts_Click; btnAddAsTanks.Click += BtnAddAsTanks_Click; btnStartImport.Click += BtnStartImport_Click; + + lstSqlAccounts.ValueMember = "Key"; + lstSqlAccounts.DisplayMember = "Value"; + btnFetchData.Click += BtnFetchData_Click; + btnImportSqlAccounts.Click += BtnImportSqlAccounts_Click; } public string MongoDbConnectionString @@ -37,6 +43,12 @@ namespace DataImporterTool set => statusImportLabel.Text = value; } + public string SqlConnectionString + { + get => txtSqlConnectionString.Text; + set => txtSqlConnectionString.Text = value; + } + public event Action OpenJsonFolderClick; public event Action AddFilesAsAccountsInfoList; @@ -45,6 +57,10 @@ namespace DataImporterTool public event Action StartJsonConvert; + public event Action FetchSqlAccounts; + + public event Action StartSqlImport; + public void ShowJsonFiles(string[] allJsonFiles) { lstAllFiles.DataSource = allJsonFiles; @@ -75,6 +91,11 @@ namespace DataImporterTool statusImportProgressBar.Value = percentage; } + public void ShowSqlAccounts(Dictionary accounts) + { + lstSqlAccounts.DataSource = new BindingSource(accounts, null); + } + private void BtnOpenFolder_Click(object sender, EventArgs e) { OpenJsonFolderClick?.Invoke(); @@ -102,5 +123,17 @@ namespace DataImporterTool { StartJsonConvert?.Invoke(); } + + private void BtnFetchData_Click(object sender, EventArgs e) + { + FetchSqlAccounts?.Invoke(); + } + + private void BtnImportSqlAccounts_Click(object sender, EventArgs e) + { + var selectedAccounts = lstSqlAccounts.SelectedItems.Cast>().Select(i => i.Key) + .ToArray(); + StartSqlImport?.Invoke(selectedAccounts); + } } } diff --git a/DataImporterTool/MainFormPresenter.cs b/DataImporterTool/MainFormPresenter.cs index 0cf0c6e..a1a4724 100644 --- a/DataImporterTool/MainFormPresenter.cs +++ b/DataImporterTool/MainFormPresenter.cs @@ -11,6 +11,7 @@ namespace DataImporterTool { private readonly IDialogService _dialogService; private readonly JsonFilesImporter _jsonFilesImporter; + private readonly SqlImporter _sqlImporter; private string[] _selectedFileAsAccounts = null; private string[] _selectedFileAsTanks = null; @@ -18,15 +19,19 @@ namespace DataImporterTool public MainFormPresenter( IMainFormView view, IDialogService dialogService, - JsonFilesImporter jsonFilesImporter) + JsonFilesImporter jsonFilesImporter, + SqlImporter sqlImporter) { _dialogService = dialogService; _jsonFilesImporter = jsonFilesImporter; + _sqlImporter = sqlImporter; View = view; View.OpenJsonFolderClick += OpenJsonFolder; View.AddFilesAsAccountsInfoList += AddFilesAsAccountsInfoList; View.AddFilesAsTanksInfoList += AddFilesAsTanksInfoList; View.StartJsonConvert += StartJsonConvert; + View.FetchSqlAccounts += FetchSqlAccounts; + View.StartSqlImport += StartSqlImport; } public IMainFormView View { get; } @@ -98,8 +103,8 @@ namespace DataImporterTool View.ClearLog(); View.SetProcessPercentage(0); - var progress = new Progress(); - progress.ProgressChanged += ProgressChanged; + var progress = new Progress(); + progress.ProgressChanged += FileImportProgressChanged; await _jsonFilesImporter.Import(View.MongoDbConnectionString, View.JsonFolderPath, @@ -108,11 +113,64 @@ namespace DataImporterTool progress); } - private void ProgressChanged(object sender, ImportProgress e) + private void FileImportProgressChanged(object sender, FileImportProgress e) { View.StatusInformation = $"{e.FilesConverted} files converted ({e.Progress}%)"; View.AppendLogRow($"Converted: '{e.AccountFileConverted}' - '{e.TankFileConverted}': '{e.AccountName}', {e.BattlesCount} battles, '{e.LastBattle.ToShortDateString()}', {e.TanksCount} tanks"); View.SetProcessPercentage(e.Progress); } + + private async void FetchSqlAccounts() + { + if (string.IsNullOrWhiteSpace(View.SqlConnectionString)) + { + _dialogService.ShowWarning("Empty SQL connection string!"); + return; + } + + View.ClearLog(); + View.AppendLogRow("Start fetching account from SQL..."); + + var accounts = await _sqlImporter.FetchAllAccounts(View.SqlConnectionString); + if(accounts != null) + { + View.ShowSqlAccounts(accounts); + } + View.AppendLogRow($"Found {accounts?.Count ?? 0} accounts"); + } + + private async void StartSqlImport(long[] accounts) + { + if (accounts == null || accounts.Length == 0) + { + _dialogService.ShowWarning("Please select at least one account"); + return; + } + if (string.IsNullOrEmpty(View.MongoDbConnectionString)) + { + _dialogService.ShowWarning("MongoDB connection string is empty!"); + return; + } + if (string.IsNullOrWhiteSpace(View.SqlConnectionString)) + { + _dialogService.ShowWarning("Empty SQL connection string!"); + return; + } + + var progress = new Progress(); + progress.ProgressChanged += SqlImport_ProgressChanged; + + View.ClearLog(); + View.SetProcessPercentage(0); + + await _sqlImporter.StartImport(View.MongoDbConnectionString, View.SqlConnectionString, accounts, progress); + } + + private void SqlImport_ProgressChanged(object sender, ImportProgress e) + { + View.StatusInformation = $"{e.Progress}% Converted"; + View.AppendLogRow($"Imported: '{e.AccountName}', {e.BattlesCount} battles, '{e.LastBattle.ToShortDateString()}', {e.TanksCount} tanks"); + View.SetProcessPercentage(e.Progress); + } } } \ No newline at end of file diff --git a/DataImporterTool/Program.cs b/DataImporterTool/Program.cs index 1a863b5..3ad86cc 100644 --- a/DataImporterTool/Program.cs +++ b/DataImporterTool/Program.cs @@ -20,6 +20,7 @@ namespace DataImporterTool var serviceProvider = new ServiceCollection() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/DataImporterTool/readme.md b/DataImporterTool/readme.md new file mode 100644 index 0000000..0155158 --- /dev/null +++ b/DataImporterTool/readme.md @@ -0,0 +1,8 @@ +#Data Importer tool + +This tool is just for import into mongoDB the old data. It can be used only once. +There are two types of old Data: +- A set of `json` Files, that contain responses from WotBlitz Api. `Account files` - response on this request - https://api.wotblitz.eu/wotb/account/info/, and `tank files` - response on this request - https://api.wotblitz.eu/wotb/tanks/stats/. +- An old SQL database from [previous version of the same project](https://github.com/DrMboga/WotBlitzStatician) + +This tool uses WinForms :) With MVP pattern.