Import module

The Import module allows you to perform a data processing process.

The data can be retrieved from a storage accessible to the app or it is possible to set up a form with the upload of the data set in the preferred format (Excel, Csv or other).

The module relies on IJobService to track the progress of the process and generate a log with the events performed.

To activate a section with an Import module, the settings in the relevant data section must be configured, specifying the settings form, if any, and the implementation class of the methods.

The module requires you to set up a class inherited from ModuleImport in which the form validation methods (optional) and the actual data processing must be implemented.

Let's now analyze the structure of a typical import module taking as an example that of newsletter subscribers.

First of all, let's set up a Form to manage the upload of the Excel file:

Where we configure the Upload control to handle files in Excel format:

We then proceed by creating the derived class and inheriting the ImportAsync method that we divide into parts here. The first part handles data recovery from Excel file

        public override async Task<List<ValidationError>> ImportAsync(Item item, Dictionary<string, object> moduleData, CancellationToken cancellationToken = default)
        {
            var providedValues = GetProvidedValues(moduleData);
            string navigationPath = navigationContext.Navigation.GetCurrentSegment().NavigationPath;

            ExcelData exchangeData = null;

            var errors = new List<ValidationError>();

            if (await jobService.IsWorkingJobAsync(navigationPath, cancellationToken))
            {
                errors.Add(new ValidationError { Name = "Import", Message = "Working job already in progress" });
                return errors;
            }

            var fileDescriptions = JsonSerializer.Deserialize<List<FileDescription>>(Convert.ToString(providedValues.FirstOrDefault(x => x.Name == "FileUpload").Value), new JsonSerializerOptions { PropertyNameCaseInsensitive = true });

            string fileName = fileDescriptions[0].FileName;
            string fileId = fileDescriptions[0].FileId;

            var uploadPath = environmentService.GetRootPath() + Path.Uploads;
            string filePath = System.IO.Path.Combine(uploadPath, fileId + ".upload");

            try
            {
                await using (var stream = new System.IO.FileStream(filePath, System.IO.FileMode.Open))
                {
                    exchangeData = await excelService.StreamToExcelDataAsync(stream, cancellationToken);
                }
            }
            catch (Exception ex)
            {
                errors.Add(new ValidationError { Name = "Import", Message = ex.Message });
                return errors;
            }

            var providedIsReplaceValues = providedValues.FirstOrDefault(x => x.Name == "IsReplaceValues");
            bool isReplaceValues = providedIsReplaceValues != null && providedIsReplaceValues.Value != null && Convert.ToBoolean(providedIsReplaceValues.Value) == true;

            var thread = new Thread(new ParameterizedThreadStart(Process));
            thread.Start(new ProcessData { ExchangeData = exchangeData, IsReplaceValues = isReplaceValues, NavigationPath = navigationPath, AppUrl = httpService.GetAppUrl() });

            return errors;
        }

From here we start the thread by passing the parameters defined in this DTO:

private class ProcessData
{
	public ExcelData ExchangeData { get; set; }

	public bool IsReplaceValues { get; set; }

	public string NavigationPath { get; set; }

	public string AppUrl { get; set; }
}

The Process method processes the data in the background:

private async void Process(object processData)
        {
            var data = (ProcessData)processData;

            bool isCancelled = false;
            string[] columns = ["Email", "AdditionalValues", "Culture", "Country", "Zone", "Interests", "Groups", "Source", "TermsName", "TermsVersion"];

            var job = await jobService.StartNewJobAsync(data.NavigationPath, (int)data.ExchangeData.Items.Count);


            // Check columns
            foreach (string column in columns)
            {
                if (!data.ExchangeData.Fields.ContainsKey(column))
                {
                    await job.AppendErrorAsync("Missing column " + column);
                }
            }

            if (job.Errors != 0)
            {
                await job.UpdateAsync();
                await job.CompleteAsync();
                return;
            }

            // Check values
            for (int i = 0; i < data.ExchangeData.Items.Count; i++)
            {
                var item = data.ExchangeData.Items[i];
                int rowIndex = i + 2;

            }

            if (job.Errors != 0)
            {
                await job.UpdateAsync();
                await job.CompleteAsync();
                return;
            }

            for (int i = 0; i < data.ExchangeData.Items.Count; i++)
            {
                var item = data.ExchangeData.Items[i];
                int rowIndex = i + 2;

                if (!await jobService.IsWorkingJobAsync(data.NavigationPath))
                {
                    isCancelled = true;
                    break;
                }

                job.Progress++;

                bool isNew = true;
                string status = "Insert";
                string providedEmail = item["Email"].ToLower();

                // Check values
                bool isValidItem = true;

                if (string.IsNullOrEmpty(item["Email"]))
                {
                    await job.AppendErrorAsync(string.Format("Email is empty at {0}", rowIndex));
                    isValidItem = false;
                }

                if (isValidItem)
                {
                    var subscriber = new NewsletterSubscriber()
                    {
                        Id = StructureDefinition.VoidId,
                        Email = providedEmail,
                        Status = SubscriberStatus.Active,
                        SubscriptionDate = timeProvider.GetLocalNow(),
                        Interests = [],
                        Groups = [],
                        AdditionalValues = [],
                        Source = item["Source"]
                    };

                    var existingSubscriber = await newsletterSubscriberService.GetSubscriberAsync(new NewsletterSubscriberFilter { Email = providedEmail });
                    if (existingSubscriber != null)
                    {
                        subscriber = existingSubscriber;
                        isNew = false;
                        status = "Update";
                    }

                    subscriber.Culture = item["Culture"];
                    subscriber.Country = item["Country"];
                    subscriber.Zone = item["Zone"];


                    try
                    {
                        await newsletterSubscriberService.SetAsync(subscriber);

                        await job.AppendLogAsync(status, providedEmail);

                    }
                    catch (Exception ex)
                    {
                        await job.AppendErrorAsync(ex.Message);
                    }
                }

                await job.UpdateAsync();
            }

            if (!isCancelled)
            {
                await job.CompleteAsync();
            }
        }

This table can be used as a reference to identify areas of the ModuleImport base class that can be customized by creating a derived class.