Skip to content

Commit

Permalink
Refactor temporary files generated by backend (#810)
Browse files Browse the repository at this point in the history 
* Upload Lift files to the temp directory before extraction
* Extract GetRandomTempDir function
* Add Project.LiftImported to track if a Lift file has already been imported (will require use of #815 on server upon release)
* Update frontend project to match backend Project
* Simplify directory and zip management
* Allow Lift files to be imported after audio is uploaded (#788)
  • Loading branch information
@johnthagen
johnthagen authored Nov 17, 2020
1 parent 4bfc0ed commit d7cf58f
Show file tree
Hide file tree
Showing 9 changed files with 209 additions and 148 deletions.
173 changes: 81 additions & 92 deletions Backend.Tests/Controllers/LiftControllerTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Backend.Tests.Mocks;
using BackendFramework.Controllers;
using BackendFramework.Helper;
using static BackendFramework.Helper.FileUtilities;
using BackendFramework.Interfaces;
using BackendFramework.Models;
using BackendFramework.Services;
Expand Down Expand Up @@ -168,19 +166,7 @@ private static string ExtractZipFileContents(byte[] fileContents)
{
var zipFile = Path.GetTempFileName();
File.WriteAllBytes(zipFile, fileContents);
var extractionPath = ExtractZipFile(zipFile, true);
return extractionPath;
}

private static string ExtractZipFile(string zipFilePath, bool deleteZipFile = false)
{
var extractionPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(extractionPath);
ZipFile.ExtractToDirectory(zipFilePath, extractionPath);
if (deleteZipFile)
{
File.Delete(zipFilePath);
}
var extractionPath = ExtractZipFile(zipFile, null, true);
return extractionPath;
}

Expand Down Expand Up @@ -284,110 +270,113 @@ public void TestRoundtrip()
_projServ.Create(proj);

// Generate api parameter with filestream
if (File.Exists(pathToStartZip))
if (!File.Exists(pathToStartZip))
{
continue;
}

using (var fstream = File.OpenRead(pathToStartZip))
{
var fstream = File.OpenRead(pathToStartZip);
var fileUpload = InitFile(fstream, filename);

// Make api call
var result = _liftController.UploadLiftFile(proj.Id, fileUpload).Result;
Assert.That(!(result is BadRequestObjectResult));
}

proj = _projServ.GetProject(proj.Id).Result;
Assert.AreEqual(proj.VernacularWritingSystem.Bcp47, roundTripContents.Language);

fstream.Close();
proj = _projServ.GetProject(proj.Id).Result;
Assert.AreEqual(proj.VernacularWritingSystem.Bcp47, roundTripContents.Language);
Assert.That(proj.LiftImported);

var allWords = _wordrepo.GetAllWords(proj.Id).Result;
Assert.AreEqual(allWords.Count, roundTripContents.NumOfWords);
// We are currently only testing guids on the single-entry data sets
if (roundTripContents.EntryGuid != "" && allWords.Count == 1)
var allWords = _wordrepo.GetAllWords(proj.Id).Result;
Assert.AreEqual(allWords.Count, roundTripContents.NumOfWords);
// We are currently only testing guids on the single-entry data sets
if (roundTripContents.EntryGuid != "" && allWords.Count == 1)
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripContents.EntryGuid);
if (roundTripContents.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripContents.EntryGuid);
if (roundTripContents.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripContents.SenseGuid);
}
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripContents.SenseGuid);
}
}

// Export
var exportedFilePath = _liftController.CreateLiftExport(proj.Id);
var exportedDirectory = ExtractZipFile(exportedFilePath, false);
// Export
var exportedFilePath = _liftController.CreateLiftExport(proj.Id);
var exportedDirectory = ExtractZipFile(exportedFilePath, null, false);

// Assert the file was created with desired heirarchy
Assert.That(Directory.Exists(exportedDirectory));
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "audio")));
foreach (var audioFile in roundTripContents.AudioFiles)
{
Assert.That(File.Exists(Path.Combine(
exportedDirectory, "Lift", "audio", audioFile)));
}
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "WritingSystems")));
// Assert the file was created with desired heirarchy
Assert.That(Directory.Exists(exportedDirectory));
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "audio")));
foreach (var audioFile in roundTripContents.AudioFiles)
{
Assert.That(File.Exists(Path.Combine(
exportedDirectory,
"Lift", "WritingSystems", roundTripContents.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedDirectory, "Lift", "NewLiftFile.lift")));
Directory.Delete(exportedDirectory, true);
exportedDirectory, "Lift", "audio", audioFile)));
}
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "WritingSystems")));
Assert.That(File.Exists(Path.Combine(
exportedDirectory,
"Lift", "WritingSystems", roundTripContents.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedDirectory, "Lift", "NewLiftFile.lift")));
Directory.Delete(exportedDirectory, true);

_wordrepo.DeleteAllWords(proj.Id);
_wordrepo.DeleteAllWords(proj.Id);

// Roundtrip Part 2
// Roundtrip Part 2

// Upload the exported words again
// Init the project the .zip info is added to
var proj2 = RandomProject();
_projServ.Create(proj2);
// Upload the exported words again
// Init the project the .zip info is added to
var proj2 = RandomProject();
_projServ.Create(proj2);

// Generate api parameter with filestream
fstream = File.OpenRead(exportedFilePath);
fileUpload = InitFile(fstream, filename);
// Generate api parameter with filestream
using (var fstream = File.OpenRead(exportedFilePath))
{
var fileUpload = InitFile(fstream, filename);

// Make api call
var result2 = _liftController.UploadLiftFile(proj2.Id, fileUpload).Result;
Assert.That(!(result is BadRequestObjectResult));

proj2 = _projServ.GetProject(proj2.Id).Result;
Assert.AreEqual(proj2.VernacularWritingSystem.Bcp47, roundTripContents.Language);
Assert.That(!(result2 is BadRequestObjectResult));
}

fstream.Close();
proj2 = _projServ.GetProject(proj2.Id).Result;
Assert.AreEqual(proj2.VernacularWritingSystem.Bcp47, roundTripContents.Language);

// Clean up zip file.
File.Delete(exportedFilePath);
// Clean up zip file.
File.Delete(exportedFilePath);

allWords = _wordrepo.GetAllWords(proj2.Id).Result;
Assert.AreEqual(allWords.Count, roundTripContents.NumOfWords);
// We are currently only testing guids on the single-entry data sets
if (roundTripContents.EntryGuid != "" && allWords.Count == 1)
allWords = _wordrepo.GetAllWords(proj2.Id).Result;
Assert.AreEqual(allWords.Count, roundTripContents.NumOfWords);
// We are currently only testing guids on the single-entry data sets
if (roundTripContents.EntryGuid != "" && allWords.Count == 1)
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripContents.EntryGuid);
if (roundTripContents.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripContents.EntryGuid);
if (roundTripContents.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripContents.SenseGuid);
}
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripContents.SenseGuid);
}
}

// Export
exportedFilePath = _liftController.CreateLiftExport(proj2.Id);
exportedDirectory = ExtractZipFile(exportedFilePath);
// Export
exportedFilePath = _liftController.CreateLiftExport(proj2.Id);
exportedDirectory = ExtractZipFile(exportedFilePath, null);

// Assert the file was created with desired hierarchy
Assert.That(Directory.Exists(exportedDirectory));
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "audio")));
foreach (var audioFile in roundTripContents.AudioFiles)
{
var path = Path.Combine(exportedDirectory, "Lift", "audio", audioFile);
Assert.That(File.Exists(path),
$"The file {audioFile} can not be found at this path: {path}");
}
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "WritingSystems")));
Assert.That(File.Exists(Path.Combine(
exportedDirectory,
"Lift", "WritingSystems", roundTripContents.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedDirectory, "Lift", "NewLiftFile.lift")));
Directory.Delete(exportedDirectory, true);

_wordrepo.DeleteAllWords(proj.Id);
// Assert the file was created with desired hierarchy
Assert.That(Directory.Exists(exportedDirectory));
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "audio")));
foreach (var audioFile in roundTripContents.AudioFiles)
{
var path = Path.Combine(exportedDirectory, "Lift", "audio", audioFile);
Assert.That(File.Exists(path),
$"The file {audioFile} can not be found at this path: {path}");
}
Assert.That(Directory.Exists(Path.Combine(exportedDirectory, "Lift", "WritingSystems")));
Assert.That(File.Exists(Path.Combine(
exportedDirectory,
"Lift", "WritingSystems", roundTripContents.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedDirectory, "Lift", "NewLiftFile.lift")));
Directory.Delete(exportedDirectory, true);

_wordrepo.DeleteAllWords(proj.Id);
}
}
}
Expand Down
24 changes: 24 additions & 0 deletions Backend.Tests/Models/ProjectTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ public void TestNotEquals()
project2.IsActive = !project.IsActive;
Assert.IsFalse(project.Equals(project2));

project2 = project.Clone();
project2.LiftImported = !project.LiftImported;
Assert.IsFalse(project.Equals(project2));

project2 = project.Clone();
project2.VernacularWritingSystem.Name = "different";
Assert.IsFalse(project.Equals(project2));
Expand Down Expand Up @@ -75,12 +79,24 @@ public void TestNotEquals()
project2.InviteTokens.Add(new EmailInvite());
Assert.IsFalse(project.Equals(project2));
}

[Test]
public void TestClone()
{
var system = new WritingSystem { Name = Name, Bcp47 = "en", Font = "calibri" };
var project = new Project { Name = Name, VernacularWritingSystem = system };
var domain = new SemanticDomain { Name = Name, Id = "1", Description = "text" };
project.SemanticDomains.Add(domain);
var project2 = project.Clone();
Assert.AreEqual(project, project2);
}
}

public class WritingSystemTests
{
private const string Name = "System 1";
private const string Bcp47 = "lang-1";
private const string Font = "calibri";

[Test]
public void TestEquals()
Expand Down Expand Up @@ -110,5 +126,13 @@ public void TestToString()
var systring = system.ToString();
Assert.IsTrue(systring.Contains(Name) && systring.Contains(Bcp47));
}

[Test]
public void TestClone()
{
var system = new WritingSystem { Name = Name, Bcp47 = Bcp47, Font = Font };
var clonedSystem = system.Clone();
Assert.AreEqual(system, clonedSystem);
}
}
}
4 changes: 2 additions & 2 deletions Backend/Controllers/AudioController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ public async Task<IActionResult> DownloadAudioFile(string projectId, string word
}

/// <summary>
/// Adds a pronunciation <see cref="FileUpload"/> to a <see cref="Word"/> and saves locally to
/// ~/.CombineFiles/{ProjectId}/ExtractedLocation/Import/ExtractedLocation/Lift/audio
/// Adds a pronunciation <see cref="FileUpload"/> to a <see cref="Word"/> and saves
/// locally to ~/.CombineFiles/{ProjectId}/Import/ExtractedLocation/Lift/audio
/// </summary>
/// <remarks> POST: v1/projects/{projectId}/words/{wordId}/upload/audio </remarks>
/// <returns> Path to local audio file </returns>
Expand Down
Loading

0 comments on commit d7cf58f

Please sign in to comment.