diff --git a/.vscode/settings.json b/.vscode/settings.json index 424179a6..d842adba 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,10 @@ "githubPullRequests.ignoredPullRequestBranches": [ "master" ], - "dotnet.defaultSolution": "disable" + "dotnet.defaultSolution": "disable", + "cSpell.words": [ + "MVVM", + "Pendrive", + "typeof" + ] } \ No newline at end of file diff --git a/AdaptiveProgramming/README.md b/AdaptiveProgramming/README.md index 941c64ad..b19e530a 100644 --- a/AdaptiveProgramming/README.md +++ b/AdaptiveProgramming/README.md @@ -10,7 +10,7 @@ The adaptive programming is presented as a catalog of language constructs, patte ## Goal -The aim of the course is to expand knowledge and improve skills of software development thanks to using technology and programming patterns to enable adaptation of the created program against the changing requirements and capabilities of the run-time production environment. This objective includes issues related to the practical knowledge of technology dedicated to postpone the decisions regarding software interoperability with development framework and the external environment. Students learn the selected technologies, design patterns, archetypes and their practical implementation in the .NET with the goal to be used while developing their own programs. +The aim of the course is to expand knowledge and improve skills of software development thanks to using technology and design patterns to enable adaptation of the created program against the changing requirements and capabilities of the run-time production environment. This objective includes issues related to the practical knowledge of technology dedicated to postpone the decisions regarding software interoperability with development framework and the external environment. Students learn the selected technologies, design patterns, archetypes and their practical implementation in the .NET with the goal to be used while developing their own programs. ## Learning outcomes @@ -97,7 +97,7 @@ The lecture is focused on the following topics: * attributed programming model, * dynamic programming, * expression representation and their translation as required by the target external system -* architecture and programming patterns related to access external data based management systems +* architecture and design patterns related to access external data based management systems * materialization to save objects state and objects graph relationship using XML, JSON, etc., * object relation mapping, * program composition using independently developed modules diff --git a/ExDataManagement/DataStreams/.Media/Blockchain.png b/ExDataManagement/DataStreams/.Media/Blockchain.png new file mode 100644 index 00000000..54fb2420 Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/Blockchain.png differ diff --git a/ExDataManagement/DataStreams/.Media/Diagram.wmf b/ExDataManagement/DataStreams/.Media/Diagram.wmf new file mode 100644 index 00000000..aec561a1 Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/Diagram.wmf differ diff --git a/ExDataManagement/DataStreams/.Media/MediaFolder.gif b/ExDataManagement/DataStreams/.Media/MediaFolder.gif new file mode 100644 index 00000000..cffd2aaa Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/MediaFolder.gif differ diff --git a/ExDataManagement/DataStreams/.Media/MediaFolderAnimated.gif b/ExDataManagement/DataStreams/.Media/MediaFolderAnimated.gif new file mode 100644 index 00000000..96f814fc Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/MediaFolderAnimated.gif differ diff --git a/ExDataManagement/DataStreams/.Media/P3.5-KryptografiaCzesc3.pptx b/ExDataManagement/DataStreams/.Media/P3.5-KryptografiaCzesc3.pptx new file mode 100644 index 00000000..a2afa7cb Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/P3.5-KryptografiaCzesc3.pptx differ diff --git a/ExDataManagement/DataStreams/.Media/Part3-N80-10-Diagram.png b/ExDataManagement/DataStreams/.Media/Part3-N80-10-Diagram.png new file mode 100644 index 00000000..3ff53c69 Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/Part3-N80-10-Diagram.png differ diff --git a/ExDataManagement/DataStreams/.Media/Part3-N80-20-Rekurencja.png b/ExDataManagement/DataStreams/.Media/Part3-N80-20-Rekurencja.png new file mode 100644 index 00000000..30a25606 Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/Part3-N80-20-Rekurencja.png differ diff --git a/ExDataManagement/DataStreams/.Media/PodpisCyfrowy.png b/ExDataManagement/DataStreams/.Media/PodpisCyfrowy.png new file mode 100644 index 00000000..614b9391 Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/PodpisCyfrowy.png differ diff --git a/ExDataManagement/DataStreams/.Media/encryption.png b/ExDataManagement/DataStreams/.Media/encryption.png new file mode 100644 index 00000000..17addf05 Binary files /dev/null and b/ExDataManagement/DataStreams/.Media/encryption.png differ diff --git a/ExDataManagement/DataStreams/DataStreams.UnitTest/FileStreamUnitTest.cs b/ExDataManagement/DataStreams/DataStreams.UnitTest/FileStreamUnitTest.cs index e7b5c00b..754347a0 100644 --- a/ExDataManagement/DataStreams/DataStreams.UnitTest/FileStreamUnitTest.cs +++ b/ExDataManagement/DataStreams/DataStreams.UnitTest/FileStreamUnitTest.cs @@ -1,6 +1,6 @@ //____________________________________________________________________________________________________________________________________ // -// Copyright (C) 2023, Mariusz Postol LODZ POLAND. +// Copyright (C) 2024, Mariusz Postol LODZ POLAND. // // To be in touch join the community by pressing the `Watch` button and get started commenting using the discussion panel at // @@ -18,20 +18,16 @@ namespace TP.DataStreams [TestClass] public class FileStreamUnitTest { - [TestClass] - public class FileTestClass + [TestMethod] + public void FileTestMethod() { - [TestMethod] - public void FileTestMethod() + string _fileName = "TestFileName.txt"; + FileExample _fileWrapper = new FileExample(); + _fileWrapper.CreateTextFile(_fileName); + using (StreamReader _stream = File.OpenText(_fileName)) { - string _fileName = "TestFileName.txt"; - FileExample _fileWrapper = new FileExample(); - _fileWrapper.CreateTextFile(_fileName); - using (StreamReader _stream = File.OpenText(_fileName)) - { - string _content = _stream.ReadToEnd(); - Assert.AreEqual(_content, _fileWrapper.FileContent); - } + string _content = _stream.ReadToEnd(); + Assert.AreEqual(_content, _fileWrapper.FileContent); } } } diff --git a/ExDataManagement/DataStreams/DataStreams.UnitTest/README.md b/ExDataManagement/DataStreams/DataStreams.UnitTest/README.md new file mode 100644 index 00000000..0e092091 --- /dev/null +++ b/ExDataManagement/DataStreams/DataStreams.UnitTest/README.md @@ -0,0 +1,20 @@ + + +# Data Streams Unit Tests + +## Table of Content + +- [1. File and Stream Concepts](#1-file-and-stream-concepts) + +## 1. File and Stream Concepts diff --git a/ExDataManagement/DataStreams/DataStreams/README.md b/ExDataManagement/DataStreams/DataStreams/README.md new file mode 100644 index 00000000..d9ec7ced --- /dev/null +++ b/ExDataManagement/DataStreams/DataStreams/README.md @@ -0,0 +1,314 @@ + + +# Implementation Examples + +## Key words + +Bitstream, File, File System, XML, XSLT, HTML, XmlSerializer, Save file, Transformation, Saving text files, Local File Systems, Open and read file, XML Schema, Common File Format, Data Access, Serialization, Validation, Visualization + +## Streaming Data Preface + +The external data is recognized as the data we must pull or push from outside of a boundary of the process hosting the computer program. In general, the external data may be grouped as follows: + +- **streaming** - bitstreams managed using content of files, or network payload +- **structural** - data fetched/pushed from/to external database management systems using queries +- **graphical** - data rendered on Graphical User Interface (GUI) + +This section collects descriptions of examples explaining the usage of the **streaming** data. + +## File and Stream Concepts + +### Operating System Context + +Using the file explorer let's get details about the `.Media` folder containing files used in the examples: + +![Fig. MediaFolderAnimated](../.Media/MediaFolderAnimated.gif) + +We have different files there, but similar descriptive data, i.e. metadata, are defined for all of them. Among these data, `Name`, `Date`, `Type`, `Size`, `Data created`, and much other information that may be useful, but the most important thing is, of course, the content of the file. + +After double-clicking on the selected file (for example `PodpisCyfrowy.png`) an image will appear. + +![PC](../.Media/PodpisCyfrowy.png) + +Here we may ask a question - how to describe this behavior? Well, a program was launched. This program must have been written by a programmer. The program opens the file as input data formatted as a bitstream, so the programmer had to know the syntax and semantics rules that were used in this file. The data contained in the file makes it possible to show the content graphically on the computer screen. This is the first example of graphical representation, but we will return to this topic later. + + + +### Program Context + +Using a code snippet located in the [FileExample][FileExample] class differences between file and stream may be explained from a program point of view. We can learn from this example that the `File` is a static class that represents the available file system and provides typical operations against this file system. The content of the file is formatted as a bitstream and represented by the abstract `Stream` class. It is an abstract class that represents basic operations on a data stream (on the bitstream), which allows mapping the behavior of various media that can be used to store or transmit data as the bitstream. From this perspective, it can be proved, that file content is always a bitstream (a stream of bytes). + +#### File Class + +Let's try the [FileExample][FileExample] class. This class is referred to by the [FileStreamUnitTest][FileStreamUnitTest] unit test. After executing the test, we can notice that the test finishes with success. But let's try to replace this caption `Today is` with the Polish translation `dziś jest` and let's execute the test again. Unfortunately, the result points out that the test hasn't passed. It may mean that the behavior of our program is different from before because we introduced Polish letters. The main reason for this problem is that I used an encoding that doesn't contain Polish letters. Precisely, a represented set of characters doesn't contain Polish letters. If we apply an encoding that supports Polish letters the test is green - it means that it passed. This means that the file's content corresponds to the stream of characters containing the national letters. Hence, it can be concluded that the bitstream becomes text after applying directly or indirectly an encoding. The set of valid characters in the stream depends on the selected encoding. + +Let us examine the behavior of files using the mentioned previously [FileExample][FileExample] class, which contains the `CreateTextFile` method. The main responsibility of this method is to save a text consisting label 'Today is' and the current date to a file. To accomplish this requirement a file is needed. The word `File` appears at the very beginning of the method. + +``` csharp + File.Delete(name); +``` + +The F12 key will take us to the definition. From the definition, it can be learned that this class is static. So there are no instances of it, we cannot create objects of this class. It is just an organization container. So this class cannot represent an individual file. It can only represent all files. It provides operations related to files, where I used one of them and this operation deletes the file whose name was passed by a parameter. + +Another interesting thing in this example is the [Open][Open] operation. The question is why to perform the open operation on a file, and what this operation would be used for. We want to save the text, but we perform open operations. Here the answer is provided by a parameter called `FileAccess`. It is an enumeration type providing all options that can be used. I selected the write operation because I want to write to this file. Well, this operation is fundamental to the use of files that we will use later, because it causes the file that is being created or opened, if it exists, to become a critical section. What does it mean? This means that no other processes can operate on this file after we have opened it. So if this file were to be used or shared by multiple applications, a lock placed by the operating system will prevent this and only allow one process to write to the file. This can have crucial consequences in a situation where for example we use a file in a hospital in which patient data is saved and is used in various places by doctors. To gain access to data at the reception, where further names are added. After someone opens the file for writing - as in this example - no one else can use the file. + +So what's important to emphasize here is that the `File` class does not represent a file. The class represents a file system. It contains operations that we can perform on any file that is available to the computer. + +The `Open` operation available in the `File` class creates an object (instance) of the `Stream` class, as follows: + +``` CSharp + using (Stream _stream = File.Open(name, FileMode.OpenOrCreate, FileAccess.Write)) + { + FileContent = String.Format(CultureInfo.InvariantCulture, "Today is {0}", DateTime.Now); + byte[] _content = Encoding.ASCII.GetBytes(FileContent); + _stream.Write(_content, 0, _content.Length); + } + +``` + +Use the `Go to definition` menu entry to visit the definition of the `Stream` class. Let me stress that it is an abstract class. It means that it can represent not only files but can also represent other resources. It is an abstract class and thanks to its various implementations we can ensure the polymorphic behavior of various objects it represents. In simpler terms, if this class represents a file in the file system, these operations will be performed by the operating system on behalf of a file system, if this class represents, for example, a computer network and operations related to a computer network, then again we will have to deal with the operations that are performed, but this time not on resources related to the file system but on resources related to the computer network. If this class represents an object that is in memory, its behavior will also be completely different than the two previously mentioned ones. We will come back to this topic by discussing various examples in which the `Stream` class responsibility has been overwritten, and inherited by classes that represent different behaviors, i.e. polymorphic behaviors of various resources that we can use to store and manage data. + +The next line does not add much to the considerations regarding the use of files to store data processed by the program. This line is where the final formatting of the string of characters to be saved takes place. The only interesting point is the possibility of choosing the syntax that will be used to write the date text form. This syntax varies and is dependent on locale selection. I chose a variant that is independent of the computer locale selection in the operating system settings. + +In the next lines of the program, we write to the file. + +``` CSharp + _stream.Write(_content, 0, _content.Length); +``` + +The file is represented by the `Stream` type, and to write data to it first, we must prepare it. It means that a bitstream must be generated based on the text to be written to the file content. + +#### Stream class + +We must be aware of how the data can be prepared. Let's look at the definition of the `Stream` type. Analyzing members that may be used to write to a variable of `Stream` type we see that all `Write' operations have parameters of type a sequence of bytes. So in this case, and in all other cases where we will use a stream to represent other data types, the data will always be formatted as the bitstream. + + + +The last thing remains to be explained, namely the close operation, which we perform on the stream. Since the open operation appeared at the beginning, in contrast, the closing operation must appear at the end. It is again fundamentally important because it closes the file, which means that the critical section is no longer needed. So, from now on, others will also be able to use this file - they will be able to open this file. Therefore, it should appear immediately after finishing working with this file. It means that we will not be going to perform further operations on this file within the program. The question is what will happen when, for example, an exception occurs in the program between opening a file and closing it. The throw statement breaks the sequence of statements to be executed. As a result, the `Close` operation will never be executed. Using modern execution environments forces this file will be closed by the environment at some point. However, this will not happen immediately and we should use different operations here, a different approach, and take advantage of the fact that `Stream` implements the IDisposable interface, which allows the use of the `using` statement. The `using` statement causes the dispose operation to be executed against the `stream` variable as the last method invocation before exiting the using visibility scope. If the stream or block of statements that is part of the using operation is interrupted, the `Dispose` operation will also be executed. Thanks to this, we can ensure that the file will be closed immediately when the next program statements no longer have access to the `stream` variable because it goes out of the visibility range. + +### XML-based Presentation + +Using bitstreams (file content) we must face a problem with how to make bitstreams human readable. First answer we know from the examples above, namely the bitstream must be compliant with a well-known application. Unfortunately, this answer is not always applicable. Therefore we should consider another answer, namely human-readable representation should be close to natural language. Of course, we have no measure here and therefore it is difficult to say whether a bitstream is close enough to natural language to be comprehensible. The first requirement for humans to understand the stream is that it has to be formatted as text. To recognize bitstream as the text directly or indirectly an encoding must be associated. An example of how to associate directly an encoding with the bitstream is the following XML code snippet: + +```xml + +``` + +The next requirement, common for both humans and computers, is a bitstream association with the comprehensive syntax rules. To make the rules comprehensive for humans the bitstream should have been formatted as a text. Finally, semantic rules should be associated with the bitstream that allows to assigning of meaning to bitstreams. + +The [ReadWRiteTest][ReadWRiteTest] sample code demonstrates how to save working data in a file containing an XML document, which next can be directly presented in other applications, like MS Word editor or Internet Explorer. In this concept, it is assumed that the bitstream formatted as XML is transformed using a stylesheet before being presented. An XML stylesheet is a set of rules or instructions for transforming the structure and presentation of XML documents. It defines how the data in an XML file should be formatted. It is the simplest way to detach a custom document content from its formatting to be presented as graphical data provided that the original document is compliant with the XML specification. + +After implementation of the [IStylesheetNameProvider][IStylesheetNameProvider] interface by the [Catalog][Catalog] class we can convey information about the default stylesheet that may be used to create an output XML file. Thanks to the implementation of the mentioned interface information about the stylesheet (`XSLT` file) is added to the XML document and can be used by any generic application to open the file and translate the content, for example [catalog.example.xml][catalogexamplexml]: + +``` XML + +``` + +This XML declaration defines an additional document that is a stylesheet document and it contains a detailed description that allows to convert the source XML document into other text-based document. If we open the source document by clicking on it, we will open a web browser and the source file will be displayed in a graphical form that can be much easier to understand by people who are not familiar with XML technology. If we look at the source of this document using the browser context menu, we can see that it is simply the earliest XML document. This document that we originally had just got transformed thanks to browser transformation. So browsers have a built-in mechanism to convert an XML file to any other text file, in this case, it is an HTML file based on a defined XML stylesheet document. + +### XML-based Validation + +If we are talking about exchanging data between different applications or between an application and a human, the issue of bitstream correctness arises. This issue should be considered on two independent levels. The first one is the correctness of the bitstream as a certain stream of signs, i.e. when the syntax rules are met. The second one is determined by the possibility of assigning information to these sequences and therefore assigning meaning to a bitstream. + +To better understand these issues, let's look at them in the context of an example [catalog.example.xml][catalogexamplexml]. The following discussion is scoped to the XML format but the presented approach should be recognized as a universal one. + +The XML (Extensible Markup Language) is a language that defines syntax rules. For example, in the mentioned above XML text after replacing the closing name of the `CD` element (by `CD1` instead of `CD` for example) we get an XML syntax error. Syntax error means that the file is not compliant with the XML standard and should not be used anymore. But after replacing the name of the opening markup of the element with the same `CD1` name then this file is correct in the context of the XML syntax. However, it is difficult to imagine that two subsequent elements will have different names but will represent the same information. So at this point, we can say that this file is compliant with the XML standard, with the XML syntax. However, it does not represent the semantics we would expect. + +Adding these attributes causes it to refer to the XML schema. + +```xml + + + + +``` + +The XML Schema allows to define additional syntax rules that will be used to check XML text against these rules. The syntax rules for the XML file must be met in a valid XML document. Hence, we can say that without the XML schema, it is just XML text. After adding schema we can define how to construct the document that is to be verified using this additional schema document. After attaching the rules described in the schema, we can therefore verify the document and assume that if the document does not comply with the schema, it means that it is not valid and should be rejected instead of being used for further processing. Thanks to this, we can ensure that documents transferred between individual applications will be verified from the point of view of their syntax rules, which should be derived from the document semantics. + +### XML-based Classes Generation + +If we are talking about communication between different remote applications, we must consider a scenario in which these applications are written in different programming languages or by different people. In this case, the problem arises of how to create types in other development environments that will represent the same information. Since we recognized the schema as a good idea to validate XML documents, i.e. XML texts, and check whether the XML text is the XML document we expect, then maybe we should turn the issue upside down and generate types in selected programming language based on XML schema. + +Let's try to see how this goal can be achieved using an example. To generate classes in CSharp, I have prepared a script [GoCS.cmd][GoCS] that uses the XSD command line application (this program is available in the Visual Studio environment). + +``` CSharp +xsd.exe Catalog.xsd /N:TP.DataStreams.Instrumentation /c +``` + +This program is used to generate the classes that we previously created manually. The classes are generated automatically based on the `Catalog.xsd` schema. We should get the result in the file created manually previously. + +As the result of executing the mentioned above script [GoCS.cmd][GoCS] the [Catalog class][Catalog.cs] is generated. A consequence of generating a new program text is that all previous modifications are overwritten - a new text is generated without caring about what was there before. Hence, it is time to recall our considerations regarding partial definitions. This is an example where we could confirm that this concept is necessary when dealing with auto-generated code because it creates new content ignoring any modifications made to this file. That's why the following message at the top of the generated file warns not to change this file. + +```txt +// +// This code was generated by a tool. +... +// Changes to this file may cause incorrect behavior and will be lost if the code is regenerated. +// +``` + +In conclusion, thanks to the application of the XML schema, XML documents can be verified against additional syntax rules, and appropriate definitions in various languages may be generated, ensuring data conversion between various technologies and different programming platforms. + +## Attributes + +### Introduction + +Attribute or annotation is a concept used in various programming languages. It is used to add metadata, comments, and supplementary information to program text. It helps enhance code readability, and document functionality and provides details for tools or compilers. Many languages may implement attributes in their own way, but the fundamental idea of associating extra information with code entities is common across many programming languages - mainly the differences refer to syntax, hence the meaning expressed as the semantics rules are almost the same. + +Apart from the definition of an attribute, it also must be possible to associate attributes with a selected language construct. This association usually is realized as a syntax constraint. For example, an attribute is added as a prefix or a decoration of a target construct. + +So the question is what is an attribute? The general answer is that it is a language construct. A programming language construct refers to a syntactical element or feature within a programming language. The constructs provide the building blocks for implementing algorithms for various problems in software development. + +To avoid meaningless explanations and get straight to the point, further explanations must be investigated in the context of program snippets prepared using the selected programming language that will be used to provide a comprehensive explanation of the syntax and semantics of the attribute definition and use. A description of the code snippets is available in this document. The examples show that attributes have broader applicability than just serialization and deserialization. However, the attribute concept is well suited to address selected issues related to the serialization/deserialization processes. + +In the selected language any attribute definition is a class derived from the `System.Attribute` class. Hence, the programming language must also provide means to instantiate this class in the context of a selected construct to which additional data has been associated using attributes. By design, the reflection mechanisms must be used to instantiate attributes at run-time. + +Depending on the development environment, attributes are crucial in controlling how objects are serialized and deserialized. They allow to provide instructions for the serialization process. The attributes help customize the serialization process to meet specific requirements. Often dedicated attributes are added to the serialization frameworks to allow the addition of expected by the specific implementation control information. Using this framework, the exposed attributes may be associated with custom definitions. + +### Profiling Development Environment + +Let's start by creating a very simple [AttributedClass][AttributedClass] example used as a starting point for the discussion on attributes. It has only one method but its functionality is not important in the context of the discussion. The method creates an object and returns it. Imagine, that after some time, we conclude that this method is not perfectly correct anymore and we want to avoid referencing it. We know it is used in many places in the program, so to preserve backward compatibility we cannot simply remove it from the program text to avoid triggering a bunch of errors. Hence, we must keep in place this definition, but we should associate additional information with code in a declarative way. This additional information should prevent it from being used in the program any further. This way we try to fix an issue by preventing referencing of inadequate code instead of replacing it. In other words, there will be no further references to it in new programs. + +We may use the `Obsolete` attribute for this purpose. To observe this attribute and the effects it causes, let's open a test window and add a test method. In the test method, we simply call the method that we previously marked with the `Obsolete` attribute and we see that the compiler now reports a warning. It is also available in the error list. Therefore, this is a clear signal that we should not use this method because it is no longer valid. + +This warning should make us use some other alternative solutions. Of course, we could use a regular comment instead. Unfortunately, this will cause us to lose the warning to avoid using this method in newly created program fragments. Based on this, we can conclude that a comment is a very good tool for **communicating with the reader** of the program text - after all, any program is a text. On the other hand, attributes are a concept to implement **communication with the compiler**. And as we will see next, not only with the compiler. + +The F12 key takes us to the definition and we see that the attribute is a class that is derived from the [Attribute][system.attribute] class. Now we can formulate a key question; whether we can define our attributes, which we may use to associate additional information with code in a declarative way to be used at run-time. + +### Attribute Definition + +To create a custom attribute I have created an additional [CustomAttribute][CustomAttribute] class. As before, it inherits from the [System.Attribute][system.attribute] base class. The main goal of it is to provide additional information related to the program content. Therefore, to define it, we need to specify the following things: + +- what additional information do we want to convey using it +- how the information is to be represented using types +- how to restrict the location of attribute usage + +The first two tasks - related to the selection of information that is to be conveyed using an attribute can be achieved by choosing how this information is to be represented (data type selection) and adding appropriate properties (value holders) that will convey this data. In this example, it is the `Description` property, which is of the `string` type. This way some descriptions expressed in natural language may be added to the target construct. Notice that also a constructor is added here, which is responsible for initializing this description when the attribute is instantiated. + +The next task of how to restrict attribute usage may be accomplished by associating an existing, dedicated attribute with a definition of a new attribute class. It is a message to the compiler. In other words, we use an existing attribute to define a new one. The [AttributeUsage][AttributeUsage] attribute is predefined by the built-in definitions of the C# programming language that allows expressing where adding a new attribute makes sense. + +And here's a crucial note about terminology. I have used the term attribute for both + +- to name a class that is derived from the [System.Attribute][system.attribute] base class +- as an identifier that is used elsewhere and surrounded between square brackets + +Maybe it sounds puzzling but it is a typical recurring reference to the joint terms. + +Let's examine the features of the newly created [CustomAttribute][CustomAttribute] class using the [CustomAttributeTest][CustomAttributeTest] test method. It just instantiates an object of this class traditionally using the `new` operator and then compares the value of the embedded property value with an expected one. This way we can prove that this class behaves like any other regular class. + +Keeping in mind that the newly created attribute is a class, let's try to use it to add additional information to the previously defined [AttributedClass][AttributedClass] class. So a linguistic construct appears, where between square brackets we will have the name of the class and additional data that we want to be associated with this class. Since this is additional data, we call it metadata; in other words, data describing data. Since in this case, the data being described is a linguistic construct, there is the text of the program - the program becomes data. The question is how this metadata may be used throughout the processing process, hence at run-time. Let's see this with an example of a unit test where we try to recover the associated data. + +From this example, we see that it can also be associated with actual parameters placed between round brackets. In other words, it looks like a method call, doesn't it? Moreover, because the name is the same as the class name it looks like a constructor call. Unfortunately, the detailed discussion of these linguistic constructs syntax is beyond the scope of the article. To possibly fill in a gap in this respect, I recommend the C# language user manual. To make the discursion generic, from now on, we will only focus on the semantics, i.e. on the meaning, of these entries. + +### Attribute Use Based Directly on Type Definition + +So let's add a test method [AttributedClassTypeTest][AttributedClassTypeTest] in a test class, in which the code will refer to [AttributedClass][AttributedClass] that has been associated with an attribute. To refer to the type the [typeof][typeof] keyword is applied. As a result of using [typeof][typeof] an object of the [Type][system.type] type is instantiated for the selected type. An object created this way contains details of the source type definition. And here we encounter reflection for the first time. Reflection, which means that we can create objects in the program that represent selected linguistic constructs. In this case, `_objectType` is a variable of the [Type][system.type] type that will contain a reference to the object representing the [AttributedClass][AttributedClass] class definition. Notice that to avoid code cloning the main test functionality is gathered in the [GoTest][GoTest] method. Then, from this object, we can read all attributes related to the selected type using the `GetCustomAttributes` instance method. Additionally, in this case, it is specified that we are only interested in attributes of the selected type. We can then determine that there is returned an array with exactly one element in it. This element is of the [CustomAttribute][CustomAttribute] type, i.e. the type we associated with the class as a class attribute. + +Therefore, we can return to the discussion about semantics, i.e. the meaning of the notation between square brackets. We see that the `GetCustomAttributes` method creates objects. Objects that are associated with selected language construct, in this case, [AttributedClass][AttributedClass]. It looks the same as if we used the `new` keyword to create an object of the [CustomAttribute][CustomAttribute] class. After creating the object it can be used as if it had been created using the `new` keyword. + +So, once again, back to the nutshell of the topic. We can ask what role this linguistic construct plays - where the class name is placed between square brackets. This class has to be an attribute, i.e. a class identifier that is derived from the [System.Attribute][system.attribute] class. We see that the main purpose of this construct is to describe the instantiation of an object, and therefore answers the question of how to create an object and populate it with the state values. This way we can conclude that it is equivalent to the constructor call, which is typically placed after the `new` operator. Here it plays the same role except that it is not part of the expression in the assignment statement. Hence it has to provide all values for the constructor and initial parameter for the members of the attribute class. + +The [AttributedClass][AttributedClass] class is preceded by the [CustomAttribute][CustomAttribute]. In the unit test, we have the [AttributedClassTypeTest][AttributedClassTypeTest] test method, which proves how to retrieve features of the definitions of this type by creating an instance of the [Type][system.type] type. The main testing stuff has been aggregated in the [GoTest][GoTest] method to reuse this functionality and avoid code cloning. This example shows that we can recover type features that are provided in the form of attributes. Additionally, we can perform operations on attributes (instances of classes derived from the [System.Attribute][system.attribute] base type) that are created as a result of the `GetCustomAttributes` operation. In this approach, the identifier of the type definition is directly used. In this code snippet, the [typeof][typeof] is an operator, which is used to instantiate an object that represents metadata of a type, utilizing the identifier of an attribute type definition. The argument to the [typeof][typeof] operator must be the name of a type definition. + +### Attribute Use Based Indirectly on Type Instance + +In the above example [AttributedClassTypeTest][AttributedClassTypeTest] there is a weak point. Unfortunately, talking about serialization/deserialization we have to implement appropriate algorithms avoiding direct reference to the type definitions because we have to assume that the definition is invisible, hence we cannot use the keyword `typeof`. In general, we must assume that the type is defined later, and it doesn't matter if it is defined milliseconds, or years later. Let's try to imagine a scenario in which we have to deal at run-time with objects whose types we do not know. + +To prepare an example that resembles the above scenario, I have added the [Siyova16][Siyova16] class with all identifiers created randomly by a password generator to stress that we should avoid using them as members of a type definition. The main idea of creating a random definition is to give the impression and stress that there is no need to refer directly to them while implementing the required functionality. To create a generic solution the reality is that we need to be prepared for a situation where referencing identifiers directly is impossible. The reflection can be applied to both cases, so we can investigate it using a simplified case. + +To continue building an example in which we will show how to operate on objects of unknown types, I have defined the [ObjectFactory][ObjectFactory] class. The main task of this class is to create objects of pseudo-random type. Precisely, the objects are only of different types, but they have one thing in common, namely, they are preceded by the same [CustomAttribute][CustomAttribute] attribute. The [AttributedClassInstanceTest][AttributedClassInstanceTest] test shows that it is possible to detect this feature without referring to identifiers associated with the object type. For this purpose, it mimics the creation of objects of various types using the dedicated [ObjectFactory][ObjectFactory] class. Regardless of the object type, the same [GoTest][GoTest] test method is performed, which checks the presence of the selected attribute. For this purpose, an enumeration type is defined in this class, in which the values are also randomly generated. It is worth stressing that there is no direct relationship between the enum identifier and the identifier of the instantiated type. + +The [ObjectFactory][ObjectFactory] method is responsible for creating objects of various types. Because it creates objects of different not related to each other types, the return value must be of the `object` type, which allows returning objects of any type. Therefore, after calling [ObjectFactory][ObjectFactory] we don't know the type of the returned instance. Hiding the type of the created objects is intended to mimic operation on unknown types. Of course, this is just a simulation for this particular example to make the example as simple as possible. I want to emphasize that the tests are solely used to demonstrate certain features and the possibility of using reflection for serialization/deserialization. + +To show how to operate on objects without referring directly to their type definitions we have to recover the features of types from their instances. To follow up, check out the example from the [AttributedClassInstanceTest][AttributedClassInstanceTest] test method. Once again, the test method instantiates a variety of types having the same feature and executes a test against this feature. + +How to recover the features of a type referring directly to this type we already know. This can be done by creating a [Type][system.type] instance for the selected type definition using the [typeof][typeof] keyword and an identifier of this definition. In the case of an object for which the type is not known for some reason, the `GetType` instance method inherited from the [Object][Object] type comes in handy. Let me remind you that this operation is inherited from the [Object][Object] base class. It is the ultimate base class of all .NET classes; it is the root of the type hierarchy. So in our case, reflection starts when an instance of [Type][system.type] is created using the `GetType` method. It should be emphasized here that based on this example, we can conclude that reflection is related even to the [Object][Object] base type. + +### Summary + +To make a summary of the discussion above, regardless of whether we have a type definition visible or we need to bother with recovering the type description from an instance instead, the common point in the process of further processing and converting the state of objects to bitstreams form and vice versa is to create an instance of the [System.Type][system.type] abstract type, which holds a detailed description of the type in concern. Because it is abstract we cannot create this instance directly and have to use the [typeof][typeof] keyword or employ the `GetType` instance method. Going right to the point, since in both cases we can reach a common point, we can have the same test method [GoTest][GoTest] to avoid text cloning. + +## Reflection + +### Preface + +Reflection is the next very useful approach that may be employed to support serialization and deserialization implementation. We can only touch on the subject of reflection, i.e. we just enter a world in which definitions in the source program become data and are processed like process data. In other words, reflection in software engineering refers to the ability of a program to examine and modify its structure and behavior during runtime. Due to the complexity of this topic, we have to limit the discussion to only selected topics useful in the context of serialization. Hence, don't expect deep knowledge related to this topic. Reflection is commonly used for tasks like recovery metadata about types, classes instantiation, method invocation, and recovering data wrapped by objects at run-time. + +So, our task is to answer the question of how to make it autonomous and automate this serialization and deserialization process. Because we have to do it in a way that allows us to avoid repetitive work. It means the mentioned functionality must be implemented in advance when we do not know the types yet. We want to offer a generic library that will be used against various types, i.e. against custom types that the user will define according to requirements of the application in concern. The only thing we can rely on and reuse is the built-in types of a selected programming language because they are immutable. + +If we need to deal with custom types, which we do not know in advance typically the following solutions may be applied. First is dynamic programming when types are created during program execution and will reflect the needs related to the data processing algorithm at run-time. The next one is the independent conversion of member values based on built-in custom functionality in new types defined at compile time. Finally, we may consider applying reflection, where type definitions created at compile time become data for the program that can be the subject of recovery metadata and reading/assigning objects state values at run-time. + +Dynamic programming is not promising and should be avoided because it is an error-prone run-time approach. Independent conversion is a design-time approach and must be considered as a serialization/deserialization method. However, it still needs custom serialization/deserialization functionality to be embedded in new type definitions, and therefore cannot be recognized as an autonomous solution. More in this respect you can get by checking out appropriate examples described in this document. Reflection allows to write the program so that the type features are recoverable and become data for the program. Reflection allows for avoiding custom implementation of the serialization and deserialization functionality. Hence, it will be described in more detail. + +The language we have selected is based on the concept of types. It is strongly typed. However, it is not the only one that uses type compatibility to check the correctness of the program at design time. However, the transition from the object-oriented world to the streaming world requires generic actions, consisting of creating generalized mechanisms for operating on data without referring to concrete type definitions. I mean the serialization/deserialization functionality must be generic without referring to type definitions, because the types may be unknown at this time. + +We want the data transformation process between object graphs and bitstreams process to be mutually unambiguous, repeatable, and autonomous. Data transformation from graph of objects form to stream form requires reading the state of these objects and the relationships between them. The reverse transformation, i.e. converting a bitstream into an object graph, requires the installation of appropriate types contributing to the graph and populating them with recovered values obtained during deserialization from the bitstream. + +The state of an object is the minimum set of values that is necessary to recreate an equivalent object. In the case of conversion from a stream to an object form, first of all, we must be able to create objects by instantiating types. If the types are instantiated, the values that have been saved as the object's state must be assigned to the internal members that are part of this object against its type. This also applies to those value holders that store information about relationships between objects, i.e. references. + +### Example + +To prepare this example, let's use the [Siyova16][Siyova16] class that has a random name and several properties defined. All the members names are also random. The main goal of using a random definition is to explain how to deal with invisible types. + +In the test project, the `ReflectionUnitTest` test class includes the [AttachedPropertyTest][AttachedPropertyTest] test method, which contains a program fragment showing how to use such a mechanism for managing a property value of an object without having to refer to its type definition. However, it should be emphasized that to implement this functionality we need to know only the name of the property and its type. This requirement must be fulfilled because the language is strongly typed. The [AttachedProperty\][AttachedProperty] class, which is the implementation holder of the reading and writing operations, is implemented in a separate library project. So obviously the library class won't be able to refer to this type because it doesn't know it - it is invisible for many reasons. We will analyze this class based on the example of the [AttachedProperty\][AttachedProperty] class. The example class [Siyova16][Siyova16] serves as a simulation of any type. + +In the [AttachedPropertyTest][AttachedPropertyTest] method I need to create a target object of type [Siyova16][Siyova16] that is to be controlled. It is worth emphasizing that creating the target object is redundant here because, in a real scenario, we should assume that the object is already created elsewhere. In the next step, a surrogate object as a wrapper of the target object is created. The surrogate object functionality is to enable reading and writing to the selected property from the target object without referring to the type of the target object. The expected behavior of the wrapper class is that the `Value` property can be assigned to and read from. These values are transferred transparently to and from the target object that is passed to it as an actual parameter of the constructor. + +The functionality enabling the possibility to get access to a selected property of a target class has been implemented as a generic library class named [AttachedProperty\][AttachedProperty]. To get more about the generic type concept check out the course or section titled [Programming in Practice - Information Computation; Udemy course, 2023][udemyPiPIC] . The library class uses a simple constructor which takes two parameters that are responsible for initializing the data members of the new object. The first parameter is used to pass references to the target object that is to be wrapped by this class. The second argument is used to pass the name of the property that is to be managed using an instance of this class. The first step of the constructor is to save the reference to the target object in the local variable. This reference will be used later. It is worth stressing that this way we cannot refer to the type of the target object. Because the target type of the object in concern is invisible reflection is engaged and the `GetType` method is used to recover the required features of the target object. The recovered description of the target object type is conveyed by a new instance of the [Type][system.type] type. Thanks to this object, in the next step we can obtain information about the property we want to write and read. This description is saved in the next local variable. The last step will be to create an intermediary property that, thanks to the previously obtained information about the target property, will allow transferring values to/from this property. + +## See Also + +- [References](./../../../REFERENCES.md#references) +- [XSL\(T\) Languages][XSLW3C] +- [Serialization in .NET][STLZTN] +- [XML Schema Definition Tool (Xsd.exe)][XSD] + +[XSLW3C]: https://www.w3schools.com/xml/xsl_languages.asp +[XSD]: http://msdn.microsoft.com/library/x6c1kb0s.aspx +[STLZTN]: http://msdn.microsoft.com/library/7ay27kt9.aspx +[system.type]: https://learn.microsoft.com/dotnet/api/system.type +[system.attribute]: https://learn.microsoft.com/dotnet/api/system.attribute +[AttributeUsage]: https://learn.microsoft.com/dotnet/api/system.attributeusageattribute +[Object]: https://learn.microsoft.com/dotnet/api/system.object + +[udemyPiPIC]: https://www.udemy.com/course/information-computation/?referralCode=9003E3EF42419C6E6B21 + +[FileExample]: FileAndStream/FileExample.cs#L19-L30 +[AttachedProperty]: Reflection/AttachedProperty.cs#L17-L46 +[AttributedClass]: Reflection/AttributedClass.cs#L17-L24 +[CustomAttribute]: Reflection/AttributedClass.cs#L27 +[IStylesheetNameProvider]: Serialization/IStylesheetNameProvider.cs#L17-L23 +[Open]: FileAndStream/FileExample.cs#L24-L29 + +[AttributedClassInstanceTest]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L46-L55 +[AttachedPropertyTest]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L57-L68 +[ObjectFactory]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L81-L101 +[AttributedClassTypeTest]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L39-L43 +[Siyova16]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L73-L79 +[typeof]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L41 +[CustomAttributeTest]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L24-L29 +[GoTest]: ../DataStreams.UnitTest/ReflectionUnitTest.cs#L103 +[Catalog]: ../DataStreams.UnitTest/Instrumentation/Catalog.xsd.cs#L21-L55 +[Catalog.cs]: ../DataStreams.UnitTest/Instrumentation/Catalog.cs#L18-L120 +[ReadWRiteTest]: ../DataStreams.UnitTest/SerializationUnitTest.cs#L42-L57 +[GoCS]: ../DataStreams.UnitTest/Instrumentation/GoCS.cmd#L1-L2 +[catalogexamplexml]: ../DataStreams.UnitTest/Instrumentation/catalog.example.xml#L1-L23 +[FileStreamUnitTest]: ../DataStreams.UnitTest/FileStreamUnitTest.cs#L19-L33 diff --git a/ExDataManagement/DataStreams/DataStreams/READMECryptography.md b/ExDataManagement/DataStreams/DataStreams/READMECryptography.md new file mode 100644 index 00000000..15b3982c --- /dev/null +++ b/ExDataManagement/DataStreams/DataStreams/READMECryptography.md @@ -0,0 +1,198 @@ + + +# Bitstream Cybersecurity + +## Introduction + +In the context of the cybersecurity of bitstreams implementation let me remind you of the following requirements we have: + +1. The first one is to ensure that all users of a source bitstream can verify that the stream has been not modified while it was being archived or transmitted. +1. The second one is to safeguard information from unauthorized access, ensuring confidentiality. +1. The third one is to confirm authorship, so all users of a bitstream can determine who created it and who is responsible for its content. This goal we call non-repudiation of the author. + +The following chapters provide more detailed descriptions of examples related to the implementation of these requirements. These requirements are implemented by applying the following cybersecurity concepts hash, encryption, and non-repudiation. + +## Hash Function + +### Fundamentals + +Let's move on to the first option for securing streams: the hash function. It is a function that transforms the input bitstream to calculate another fixed-size unique bitstream. A collision in a hash function occurs when two different inputs produce the same hash value as output. The next feature of the received output bitstream is that the reverse transformation, i.e. recovering the source bitstream is practically impossible. One way to use such a function is to associate this hash value with the bitstream we want to protect. Then the hash value can be used to check whether the bitstream has not been modified in the meantime by calculating this function again and comparing the result with the associated hash value with the source bitstream if the expanded bitstream is archived or sent from one place to another. A certain drawback of this solution is that the algorithms for these functions are widely known, so if a "man in the middle" wants to modify the source bitstream, they can modify the source bitstream and recalculate a new value of the hash function for the previously modified bitstream. + +Anyway, there are a few scenarios where this approach makes sense. Well, for example, the value of the hash function may be entered into the next bitstream called block, and a chain protection is created. The next block, which is also a bitstream, containing this hash value and pointing to the previous block means that we cannot modify the previous block because the value of the hash function is stored in the next one. This type of chain security is called blockchain and is used widely to protect against double-spending on crypto-currencies, for example, Bitcoin (fig below). + +![Blockchain](../.Media/Blockchain.png) + +Blockchain security helps ensure that if someone wants to modify one of the blocks in the chain, they must modify all the blocks that have been attached to that chain later. Of course, this is still possible, so further safeguards are needed. Among other things, the growth rate of this chain, i.e. the speed of adding subsequent blocks to the chain, is greater than the possibility of modifying fragments of the chain. This topic is far beyond the scope of this document, but if you are interested in getting more I encourage you to check out a dedicated GitHub repository [NBlockchain][NBlockchain]. There is a practical example of how to implement such a chain. + +### Example + +So let's see how the hash function works and how it can be used in practice. In the [CryptographyHelpersUnitTest][CryptographyHelpersUnitTest] class, two unit tests have been prepared. They use the [CalculateSHA256][CalculateSHA256] method defined in the library. It is worth emphasizing once again that the argument of a hash function is always a bitstream. But obviously, the hash function may also be used for text, namely a bitstream for which an encoding has been defined. In the `CalculateSHA256Test` method, we have to protect a password. It is a string of random characters. Password may be associated with syntax and semantics to make it easier to remember but, fortunately, these syntax and semantics rules have no impact on the hash calculation. In this method, instead of a bitstream, we have a stream of characters compliant with the string type. The Alt+F12 key will take us to the definition of the [CalculateSHA256][CalculateSHA256] method. The input parameter of this method is a sequence of characters of the `string` type, but the hash function operates on an array of bytes, therefore we must transform this string of characters into a string of bytes. To do this, we need to have associated an encoding. In the case of the method under consideration, this is `UTF8`. This is the first yellow light that should light up because everyone who will use the result of the hash function to check the correctness of the input string must use the same encoding format (UTF8 in this case). If someone uses a different encoding, the hash function cannot necessarily be used to check the consistency of the input text. To be able to calculate the hash function in the [CalculateSHA256][CalculateSHA256] method, we need to create an object of the `SHA256Managed` class available in the language library. Since it implements `IDisposable`, I used the using statement. + +In the next line: + +``` csharp +return (BitConverter.ToString(hashValue), Convert.ToBase64String(hashValue, Base64FormattingOptions.InsertLineBreaks)); +``` + +a bitstream generated by the hash function is converted into two text forms. The `BitConverter.ToString` converts the numeric value of each element of a specified array of bytes to its equivalent hexadecimal representation. The second form is a string with a notation consistent with the hexadecimal code compliant with the `Base64` standard. + +`Base64` is a binary-to-text conversion. The output of this conversion represents binary data in an ASCII string format. It is commonly used in scenarios where binary data needs to be stored or transferred as text. This conversion method `Base64` is available in the language library and has many overloads. They all implement the RFC 2045 standard. And here another yellow flag should be raised because it is not the only standard that defines the `Base64` conversion. Moreover, based on the RFC database, it is easy to conclude that several RFC documents previously defined this conversion. So we can expect that this standard has been modified over time. Therefore, the question is about backward compatibility and the lifetime length of the calculated hash value if it is saved as text compliant with the `Base64` standard. It may turn out that the input string has not changed, but in the meantime, the implementation of the `Base64` conversion has changed and therefore using this string for validation is useless. + +In unit tests methods, we have two assertions, which compare the result returned by hash calculation methods with defined hard-coded text. If the encoding changes when converting the input string of characters and when the implementation of the conversion to hexadecimal text or `Base64` changes, we can expect that these assertions and invariants will not be true and the test will end with an error. And we also have to consider this as another yellow flag that has to be raised. In other words, the use of a string, although convenient, unfortunately, has the consequence that this conversion from a bitstream to text compliant with the string type does not always have to be the same and may change over time. So why use it; someone may ask. In that case, wouldn't it be better for us to base it on a sequence of bytes? Well, we cannot always attach such a sequence of bits to the text; if it is e-mail, for example, then the email system has strictly defined characters that it can use to control data flow. Hence, it has to be taken into consideration the fact that attaching such a raw bitstream could have invalid characters causing problems with the correct operation of the email system. Therefore, conversion to text is sometimes necessary, but you need to remember these caveats. + +## Encryption + +### Fundamentals + +Encryption is a reversible bitstream transformation function into another bitstream. The transformation or scrambling function rearranges or modifies the order of bits in a bitstream. This function is designed to introduce complexity and randomness into the data, making it difficult for unauthorized parties to interpret or understand without the appropriate decryption process. The goal is to enhance the security of the information being transmitted or stored. After encryption, the encoding, syntax, and semantics rules no longer apply to an output bitstream. So, as a consequence, no information can be associated with the obtained this way bitstream. The diagram below shows how it works. + +![Encryption](../.Media/encryption.png) + +The result of this encryption function (`Fe`) depends on the `K1` key. The `K1` key is also a bitstream. The disadvantage of this solution is that the resulting bitstream is always the same because the `Fe` is a function. This is easily fixed after adding a few randomly generated bytes to the input stream; the so-called nonce. Thanks to this, the result will be different each time even if the key `K1` is the same. This approach protects against the possibility of repetition, i.e. using the same bitstream even without understanding its meaning. To perform the reverse operation, i.e. restore the source bitstream that was originally encrypted, a decryption operation must be performed. For this, we will need the second key marked `K2` in the drawing above. If nonce has been added it is removed before the bitstream is ready for reusing. + +If K1==K2 we have symmetric encryption, otherwise we call the encryption asymmetric. + +Symmetric encryption employs a single key for both encryption and decryption operations. The same key is used by both interoperable parties, providing a more efficient process than asymmetric encryption. However, secure key exchange becomes crucial for maintaining confidentiality in symmetric encryption. The next problem with symmetric encryption is scaling. The scaling problem with symmetric encryption arises when a large number of parties need to be part of interoperability securely. In this scenario, each pair of interconnected parties requires a unique symmetric key for encryption and decryption. Managing and securely distributing a large number of keys becomes challenging, impacting the scalability of symmetric encryption in a practical setting. + +Asymmetric encryption, also known as public-key cryptography, involves a pair of keys, namely a public key used for encryption and a private key for decryption. Bitstreams encrypted with the public key can only be decrypted by the corresponding private key, ensuring secure interoperability. On the other hand, bitstreams encrypted with the private key can only be decrypted by the corresponding public key. + +### Symmetric Cryptography Example + +It is proposed to analyze the encryption and decryption process using the [EncryptDecryptDataTest][EncryptDecryptDataTest] test method defined in the [CryptographyHelpersUnitTest][CryptographyHelpersUnitTest] class. In this method, symmetric encryption is used that implements the 3DES algorithm. We will encrypt the selected XML file [catalog.example.xml][catalog]. The test method must be preceded by an attribute that ensures all necessary files are copied to the test workspace before this method is invoked. First, we check whether this file exists. An assertion must always be true indicating that the file exists. We will save the encrypted result in another file. If this file exists, it is deleted. `ProgressMonitor` is a local class that will be used to track the progress of encryption and decryption progress. We will come back to this class shortly. The next step is directly related to encryption. + +The purpose of the following instructions is to create an object that generates an encryption/decryption key. + +```csharp-interactive + TripleDESCryptoServiceProvider _tripleDesProvider = new TripleDESCryptoServiceProvider(); +``` + +The key consists of two independent parts and can have different lengths depending on your needs. For default parameters, the length is 192 bits. + +The next method [EncryptData][EncryptData] encrypts the input file and places the result in the output file. But to perform the encryption operation, we still need to pass two parameters, a key and an initialization vector. Please note that both, the initialization vector and the key are arrays of bytes, they are simply bitstreams. Where these keys are generated is important because access to these keys guarantees selective access to the bitstream meaning. This means anyone who has both the key and the initialization vector will be able to decrypt the encrypted bitstream. Although this example doesn't show it, we should take care of the distribution of keys and, of course, the initialization vector. We can treat these two things as one whole. + +So let's see how the encryption may be applied. In this example, we are encrypting an XML file used to save a directory containing CD descriptions. Let me stress that the content of the file has no impact on the encryption/decryption process. The most important thing is that it is just a bitstream. The [EncryptData][EncryptData] encryption method has the following parameters: input file name, output file name, key, and initialization vector. Next, [dependency injection][DI] is used to allow the calling method to keep track of the process as the encryption process happens in stages. + +First, we open the file to read the input bitstream, which contains the source data that is to be encrypted. The data is encrypted step by step using small chunks and is preserved in a buffer that has a predetermined length. In this case, the buffer is 100 bytes long. Encryption requires the creation of a bitstream complying with the `CryptoStream` type. To create its instance, we will need an object for which we pass the key and an initialization vector. The encryption itself is performed using the `Write` method, which writes bytes from the local buffer to the `CryptoStream` object. After saving, we inform the invoking party about the process progress returning the total number of bytes that are saved in this step to monitor the progress of work. Please note here that although the source file is a text file, we treat it as a bitstream. Associated encoding with this file content is not important and can be neglected. So, to read it, we create an object of the `FileStream` type because, as it was said while encrypting the encoding of the input file is irrelevant. In other words, encryption is always performed for the bitstreams. The encryption process ends when we read zero bytes into the buffer. + +Then in the test method, after encrypting the source file, we check that the output file exists. There is an assertion that checks that this file exists and finally, we check if the number of bytes in the source file is equal to the number reported and written in the output file. + +And now we move on to the step where the file is decrypted. That one we created. The entire procedure is carried out in the [DecryptData][DecryptData] method. This is again the library method. We pass similar parameters to it. Let me stress that to succeed the same key and the initialization vector that was used earlier have to be passed. Of course, in a real scenario, decryption is performed in a different location usually by a different computer, or even in a completely different place in the world, therefore we must ensure that whoever performs the decryption process in the location where the decryption process is to be carried out need the same key and the initialization vector used while the stream is encrypted. + +So let's take a look at how the decryption procedure is implemented in the [DecryptData][DecryptData] method. It is easy to note that it is very similar to the encryption method. Again, we treat the output file as a bitstream opened for writing so that we can store the decrypted bytes. We will carry out the entire process step by step using small chunks preserved in a buffer, which has the same length as the previous one. What is important is that we must have an object of the `TripleDESCryptoServiceProvider` class that provides the same key and the same initialization vector that was previously used. This time, `CryptoString` will have a `mode` parameter indicating that it will be used to read a file content, so it will generally operate by decrypting the content of the specified file. In the [DecryptData][DecryptData] we have created an object that is responsible for performing decryption operations. Again, we end the process when we have read all the bits from the file containing the encrypted bitstream. We report the progress of this process using the `Report` method. The operation finishes when everything has been saved to the output file. Of course, the output file is automatically closed thanks to the `using` statement. For the sake of simplicity, in the [EncryptDecryptDataTest][EncryptDecryptDataTest] test method, the only correctness validation of the encryption/decryption round trip process is that the length of the file after decryption is equal to the length of the input file that is the source file. + +### Conclusion + +We have already learned that there are two types of encryption. In the examples discussed in this section, the symmetric encryption method of the bitstreams was the subject of examination. Asymmetric encryption will be the subject of the next section covering digital signature generation and validation. + +In the symmetric encryption, the encryption and decryption sides use identical keys. From the point of encryption up to decryption, the bitstream is highly likely to be secured because no information can be associated with it. In other words, it cannot be used to recover the information it originally represents though the encrypted meaningless bitstream is available. + +In the next part, we move on to asymmetric encryption. Precisely, not the encryption itself because the performance of asymmetric encryption is not enough hence it is only used in selected scenarios. The next section explores examples illustrating digital signature scenarios in which asymmetric encryption can and should be used. Asymmetric encryption is also used to distribute a session key securely. The session key for communication encryption is a temporary cryptographic key used to secure communication between parties during a specific session. It is generated for a short duration and provides a secure means for encrypting and decrypting bitstreams exchanged between the communicating entities. Session keys can be securely exchanged using asymmetric cryptography, where each party has a pair of public and private keys. The public keys can be exchanged openly, while the private keys are kept secret. + +## Non-repudiation + +### Fundamentals + +Digital signatures are widely used in electronic transactions, software distribution, and other scenarios where ensuring the origin and integrity of bitstreams is crucial. Let's check how asymmetric encryption could be implemented in this subsection. First of all, I propose to deal with the confirmation of authorship. This issue has been associated with the topic of ensuring bitstream integrity. We have already learned that there are two types of encryption. In the examples discussed in this chapter, only the asymmetric encryption method of the bitstreams is the subject of examination. Symmetric encryption has been the subject of the previous chapter covering the confidentiality of bitstreams. + +So let's move on to how a digital signature works, and how we ensure that the document's author cannot deny that he is the author. + +### Compliance with Domain-specific Language + +If a bitstream to be signed is compliant with domain-specific language (for example XML) any inserted text to this bitstream must not break compliance with this language. For example, consider the [catalog.example.xml][catalog] document that we already used in examples. Let's try to add a free-formatted text at the end of this document, for example, a previously calculated hash value expressed as hexadecimal text encoded using ASCII standard. Well, of course, we can easily predict the result. There is a syntax error reported, hence it can be stated that this document is no longer an XML document. Because the syntax is not correct it is not possible to recover the meaning of this document as one whole including added text. It is simply a free text and is not suitable for further processing when we expect the document to follow XML syntax rules. + +What can we do? We can surround this text with an element markup, which is called for example `Hash`. As a result, we no longer have an XML syntax error, but we do have an error that such an element does not exist according to the schema we have defined. We can dumb down this document again and remove references to the schema, which defines what an XML document should contain. But this again leads to further consequences, such that if we expect that this document is compliant with a certain schema, then, as a consequence, this document is rejected because the schema is not defined for it. I would like us to remember this when following the method of implementing a digital signature. It will be vital to us. + +### Signing Process + +The following diagram shows how to implement authorship non-repudiation of a bitstream + +![Digital Signature](../.Media/PodpisCyfrowy.png) + +In the first step, we calculate the hash, just like before. But then we encrypt this hash using a private key, which is assumed to be assigned to a certain identity, which is to be exclusively at the disposal of this identity. So, at least theoretically, no one else can use this key. If we encrypt the hash using a private key and asymmetric algorithm, the result is called a signature. We can therefore attach this signature to the original bitstream, archive, or send the result to another place over a network. + +To check the bitstream integrity and authorship at the final location, we can first recalculate the hash value for the part that constitutes the source bitstream. This hash should be the same as the one recovered from the signature. To recover the hash value, namely, determine its initial value, the signature can be decrypted using the public key. The public key is associated with the private key (both are the keys pair), and we will then obtain the decrypted hash value calculated by the author of the bitstream. To check the correctness of the bitstream before further calculation, we can now compare the decrypted initial hash value with the hash value that is calculated after receiving the bitstream. If these two hash values are the same, it means that the input bitstream has not been affected because the hash value is still the same. Since we used a public key that is paired with a private key, we can also conclude that a specific identity created this stream. For any other identity, this decryption operation will not produce an identical hash value. + +And now the last thing is how to ensure non-repudiation. How to ensure that the person who originally signed this bitstream will not say after some time that it is not him/she, that it is someone else? We can do it only after ensuring that the public key has been provided by a public benefit organization, just like an ID that confirms our identity. This means that we trust a certain organization that issued this key. This key is made available to us in the context of personal data, data that describes the identity, and therefore, based on this trust, we can conclude that this is a specific person, a specific identity. + +### Creation of Keys Example + +Let's move on to discussing how to implement this scenario using program text. As we can see from the description of this scenario, one of the important problems we have is creating and distributing keys. Hence, the first test method [CreateRSACryptoServiceKeys][CreateRSACryptoServiceKeys] is an example of how to generate keys and to point out how these keys may be distributed as an XML text. Of course, the topic related to key distribution - in general - is far beyond the scope of this section, therefore let me encourage you to check out other publications at this point. In this test, I use a method that generates keys. Let's go to its definition and see that in the first step an object `RSACryptoServiceProvider` is created for which we define the key length. This is a parameter that also determines the strength of security, but at the same time, it has some negative impact on the performance of this process. Depending on the equipment we have, this number should not be outsized here. + +Once this object is created, the keys are generated. We can use these keys and we have three options. First, we can return the keys as an `RSParameters` object. An object of this class contains both private and public keys but is intended only for use inside the application. It is not intended to be used for key distribution outside of the application hosting realm. The next two lines show how to generate XML text that contains the keys. The XML form is suitable for archiving or distributing the keys over the network. Anyway, in the investigated sample, all three forms of keys are returned as a result of this method. It is a redundant solution to show only possible implementations. + +Let's go back to the [CreateRSACryptoServiceKeysTest][CreateRSACryptoServiceKeysTest] test method, where we check that the first variable is not `null`, so an object of the `RSAParameters` class is returned. We further check that the content of the XML documents - they are simply a text - that contains only the public key [PubliKey.xml][PubliKey] and the one that contains the public key and private key, are not the same. From the point of view of testing, these operations are not important, but they show how the `CreateRSACryptoServiceKeys` method works. + +The XML document that contains both the public and the private keys is located in the file [PubliPrivateKeys.xml][PubliPrivateKeys]. Of course, in the case of a private key, identity information is not important because, by design, the private key is only used by the owner. + +The situation is different when we have a document containing only the public key. [PubliKey.xml][PubliKey] is an XML document that contains only the public key. Since this key will be used by third parties (bitstream users), by design, the distributed document must contain information about the identity to which this public key is associated. Of course, this is not fulfilled here. For this to be true, information about the public key must be added to another document called a certificate. A certificate is a document that has just been issued by a trusted organization. The organization is an office that certifies with its signature that the certificate is authentic and contains correct information. From the certificate itself, we can find out what identity the public key is assigned to. Unfortunately, discussing these issues in detail, as I said earlier, is far beyond the scope of this document. + +### XML Document Signing Example + +Let us now discuss how to implement the operation of signing an XML document and how to encapsulate the obtained signature in this document so as not to violate the rules of syntax control consistent with its schema. We are using XML but the same approach is available for any domain-specific language. First, we will need an input file that will serve as a signed source document. For this purpose, the file [catalog.example.xml][catalog] is used. We will also need the keys. We will use the private key to sign the document, precisely to encrypt the hash value calculated for the initial bitstream. We will use the public key to check the validity of the signature, precisely to decrypt the attached signature. + +The signing operation is performed in the [XmlSignatureTest][XmlSignatureTest] test method. This operation is implemented in the [SignSaveXml][SignSaveXml] method to which we passed the source document to be signed, the keys that will be used for signing, and the name of the document where the signed document is to be saved. In this method, apart from checking the correctness of the input arguments, we create an instance of the `RSACryptoServiceProvider` class, which is to be used to create a signature so that we can place the signature in this document. Signing itself means that we add a signature in the last instruction. To create this signature, we use the keys that we passed while invoking the method, so this instance is initialized with the keys that were passed here so that the entire signing process takes place using the keys that will be further used to check the signature. Finally, the signed document is saved to a file. + +So let's return to the [XmlSignatureTest][XmlSignatureTest] test method. We assume that we have already signed and saved the document in the file and now we can move on to discussing the methods that check the correctness of this document. There are two overloads of the [LoadVerifyXml][LoadVerifyXml] methods. Calling the first overload, we do not transfer the keys. It is worth emphasizing that the document is loaded and checked using the public key stored in it. With this solution, we do not have to bother providing the public key. Of course, with this type of checking, we cannot confirm the author's identity because anyone can enter such a public key. The only thing we can do is validate whether the document is consistent against this key. The second overload of this method uses the already passed keys and initializes the `RSACryptoServiceProvider` object, which is used to check the document authorship. + +Finally, let's look at the signed XML document [SignedXmlFile.xml][SignedXmlFile]. We can see that the `Signature` element has been added. This document is currently erroneous because it is incompatible with the declared document schema. To fix it, the `Signature` element has to be removed from the XML document just after validation against the signature, and before using this document, for example for a deserialization operation; i.e. creating a graph of objects based on it. + +A `Signature` element complies with the XML Digital Signature standard, namely [XML Signature Syntax and Processing Version 1.1][XMLS] issued by W3C in 2013. It is used to encapsulate digital signatures within an XML document. The `Signature` element contains additional information including the cryptographic signature value and details about the key used for signing. Thanks to this it can be easily removed from the XML document before further processing. + +### Conclusion + +In this part, we move on to asymmetric encryption. Precisely, not the encryption itself because the performance of asymmetric encryption is not enough hence it is only used in selected scenarios. This section explores examples illustrating digital signature scenarios in which asymmetric encryption can and should be used to create a safe hash value interchange channel. Asymmetric encryption is also used to distribute a session key securely. The session key for communication encryption is a temporary cryptographic key used to secure communication between parties to establish a secure session. It is generated for a short duration to be used to establish a secure session allowing for encrypting and decrypting bitstreams exchanged between the communicating parties. Initially, session keys can be securely exchanged using asymmetric cryptography, where each party has a pair of public and private keys. The public keys can be exchanged openly, while the private keys are kept secret. + +## See Also + +- [References](./../../../REFERENCES.md#references) +- [Programming in Practice - Information Computation; Udemy course, 2023][udemyPiPIC] - Information Computation means a process engaging a computer (a physical device) to process information as a series of actions or steps taken to achieve a particular result or help fulfill a task. The main challenge is that information is abstract. Precisely speaking, it is a kind of knowledge that cannot be processed directly by any physical device. Generally speaking, To resolve this inconsistency two main topics are covered. The first one refers to selected aspects of information modeling using types as descendants of a coding system. The second one covers program architecture design patterns to improve the design and deployment of the computer behavior description using a program implementing an algorithm. +- [Programming in Practice - Executive Summary; Udemy course; 2021][udemyPiPES]; The course explains the role of this repository as the extended examples storage that is a foundation for the Programming in Practice paradigm. The course is for all serious about the improvement of the software development skills education methodology. +- [Programming in Practice; GitBook eBook](https://mpostol.gitbook.io/pip/) - The content of this eBook is auto-generated using the Markdown files collected in this repository. It is distributed online upon the open access rules. +- Connect to [Programming in Practice Education LinkedIn Group][LinkedInPiP] +- [Generic implementation of the Blockchain agent in .NET][NBlockchain] +- [XSL\(T\) Languages][XSLW3C] +- [Serialization in .NET][STLZTN] +- [XML Schema Definition Tool (Xsd.exe)][XSD] +- [Generic implementation of the Blockchain agent in .NET][NBlockchain] +- [Dependency Injection][DI] +- [XML Signature Syntax and Processing Version 1.1][XMLS]; W3C Recommendation; 11 April 2013 + +[NBlockchain]: https://github.com/mpostol/NBlockchain#nblockchain +[udemyPiPIC]: https://www.udemy.com/course/information-computation/?referralCode=9003E3EF42419C6E6B21 +[udemyPiPES]: https://www.udemy.com/course/pipintroduction/?referralCode=E1B8E460A82ECB36A835 +[LinkedInPiP]: https://www.linkedin.com/groups/7478959/ + +[XMLS]: https://www.w3.org/TR/xmldsig-core1/ +[DI]: https://www.udemy.com/course/information-computation/?referralCode=9003E3EF42419C6E6B21 +[XSLW3C]: (https://www.w3schools.com/xml/xsl_languages.asp) +[XSD]: (http://msdn.microsoft.com/library/x6c1kb0s.aspx) +[STLZTN]: (http://msdn.microsoft.com/library/7ay27kt9.aspx) + + +[CalculateSHA256]: Cryptography/CryptographyHelpers.cs#L23-L31 +[LoadVerifyXml]: Cryptography/CryptographyHelpers.cs#L161-L195 +[DecryptData]: Cryptography/CryptographyHelpers.cs#L62-L86 +[EncryptData]: Cryptography/CryptographyHelpers.cs#L33-L60 +[CreateRSACryptoServiceKeys]: Cryptography/CryptographyHelpers.cs#L88-L101 +[SignSaveXml]: Cryptography/CryptographyHelpers.cs#L111-L146 + + +[EncryptDecryptDataTest]: ../DataStreams.UnitTest/CryptographyHelpersUnitTest.cs#L49-L76 +[CryptographyHelpersUnitTest]: ../DataStreams.UnitTest/CryptographyHelpersUnitTest.cs#L23-L140 +[XmlSignatureTest]: ../DataStreams.UnitTest/CryptographyHelpersUnitTest.cs#L91-L118 +[CreateRSACryptoServiceKeysTest]: ../DataStreams.UnitTest/CryptographyHelpersUnitTest.cs#L79-L87 +[PubliPrivateKeys]: ../DataStreams.UnitTest/Instrumentation/PubliPrivateKeys.xml#L1-L11 +[PubliKey]: ../DataStreams.UnitTest/Instrumentation/PubliKey.xml#L1-L5 +[catalog]: ../DataStreams.UnitTest/Instrumentation/catalog.example.xml#L1-L23 +[SignedXmlFile]: ../DataStreams.UnitTest/Instrumentation/SignedXmlFile.xml#L3-L42 diff --git a/ExDataManagement/DataStreams/DataStreams/READMESerialization.md b/ExDataManagement/DataStreams/DataStreams/READMESerialization.md new file mode 100644 index 00000000..a0ee4ef3 --- /dev/null +++ b/ExDataManagement/DataStreams/DataStreams/READMESerialization.md @@ -0,0 +1,178 @@ + + +# Objects Serialization + +## Introduction + +From the previous considerations, we know that serialization/deserialization is the transformation process of data wrapped by an object from/to a bitstream form. These operations should be implemented as generic ones. It means that they must not depend on the type of the serialized/deserialized object because they should be offered as a generic library solution to allow multi-time usage against custom types. This process must start with recovering a set of value-holder members constituting the state of an object. Let me stress, that to provide a generic solution, this mechanism must not depend on the object type. + +Let me stress again that we have two issues that we need to resolve. The first issue is a selection of the value-holder members that should be included in the resulting stream. The second is reading the values for these selected value-holder members without directly referencing the type definition. + +Talking about serialization/deserialization we must answer the question of how to build universal and stand-alone libraries that will allow you to transfer data wrapped by an object to a bitstream and recover it from a bitstream to populate instantiated types. In other words, reading and writing values to its selected members without directly referring to its type. The main problem is that if the concrete type definition is not visible we don't have access to its members. We will analyze the following two typical approaches to implement this algorithm: + +- **self-controlled** - the type exposes functionality that enables reading from and assigning to the type members contributing to the instance state +- **attributes and reflection** - metadata added by attributes to select state contributors and reflection that enables reading from and assigning to properties constituting the state + +### Self-controlled Serialization + +The first approach, compliant with the above scenario, is to implement access to object state values internally by a custom type. An example of this approach is presented later in this section. It is based on internal reading and assigning operations of the values creating the object's state in compliance with the object type definition. This way, it is possible to avoid the need for employing reflection. Instead, the [ISerializable][iserializable] interface has to be implemented. + +```aspx-csharp +public interface ISerializable +{ + void GetObjectData(SerializationInfo info, StreamingContext context); +} +``` + +This interface acts as a contract between the target but custom class to be serialized and the class that implements the final transformation of a collection of values to a bitstream and by design, implements this algorithm without detailed knowledge about the target type. Only this interface is in common. We must be aware that the proposed solution is not perfect. There are still many issues that have been left unsaid. So let's start by systematizing the shortcomings of this proposal. + +The first issue that we must address is the full autonomy of the serialization and deserialization process. In this approach, we must manually ensure that the appropriate values constituting the state of the target object are saved in the dedicated array, which is passed on to be written to the bitstream. It means that partially this functionality must be implemented by the custom type in compliance with the [ISerializable][iserializable] interface instead of being provided by a generic library. + +The second issue related to the self-controlled approach to access values constituting the state of an object is the necessity of harmonization of the custom operations carried out during the serialization with the operations carried out during deserialization. From the mentioned examples we learn that two separate pieces of custom code are responsible for implementing this responsibility, and therefore any modification in one code piece must be mirrored in the other piece. This can lead to errors if this is not the case. + +The main benefit of this technique is the lack of necessity to additionally determine the state of the object because this operation is postponed to the serialization mechanism itself. This mechanism must be implemented manually by the author of the type so it could be assumed that only vital values are transferred to the bitstream format and recovered from the bitstream. + +Using self-controlled determination of a set of values contributing to the object state means splitting the functionality between the type to be subject for serialization/deserialization and library functionality, which is responsible for saving the value of the selected members independently in a bitstream. This solution requires that the type to be serialized must be prepared to read/write values from the members and create a table against an interface that is a kind of contract between both parties responsible for implementing the serialization/deserialization functionality. The main problem is that the type of concern must be prepared against the contract defined by the implemented interface. + +Examples illustrating serialization using self-controlled access to values contributing to an object state are described later in the section [Self-controlled Serialization][self-controlled-serialization-example] + +### Attributes and Reflection + +Instead of using a self-controlled data access approach, the reflection may be employed to read and write values contributing to the object state. This way there is no custom code related to selecting, reading, and writing state values. To select only necessary values the following convention may be applied. It says that the state of the object is constituted by all the values that can be obtained by reading the public properties that have both getter and setter. So from this, you can read the current value and assign new ones. If this convention applies to the target object and all indirectly referenced ones we can state that the graph of objects is ready for serialization and deserialization using reflection. What is very important is to ensure symmetry between serialization and deserialization. This means that using reflection there is no need to add any dedicated functionality to the target class related to serialization and deserialization. It addresses the error-prone self-controlled data access responsibility of a type. + +The rule that in the output stream all the values must be saved, which can be retrieved from public properties, and which have both getter and setter cannot be used uncritically. We also need to consider the case when such properties exist, but for some reason, we do not want to save their values in the output stream - they don't contribute to the object state. A solution to this problem may be based on applying dedicated attributes. It means that in practice properties of this kind are preceded by a selected attribute. For example, it may be `XMLIgnore`. + +``` csharp +public class XmlIgnoreAttribute : Attribute +``` + +It indicates that you must use all public properties having a getter and setter, except those preceded by this attribute. + +In this solution, the question is whether we can ensure the symmetry of the serialization and deserialization operations. The answer is yes because reading data and writing data functionality are side by side in the same place using the same property. + +Recapping, from the above, we may learn how to use attributes and reflection to ensure full autonomy of the serialization process and harmonize the behavior of converting objects to a stream and stream to objects. Autonomy in this context means that the reflection is employed to implement a library and, as a result, the conversion process can be performed without dedicated custom code embedded in the type of objects to be serialized and deserialized. + +Reflection-based serialization is a technique in software engineering where the internal structure of an object is recovered and internal data is serialized or deserialized based on metadata available at run-time related to the type of the object. This approach allows for dynamic transferring of object state to bitstream without explicit configuration. The main outcome of this approach is that in the target type (subject to serialization) there is no need to create dedicated code that is used to implement this operation. So we can say that reflection enables us to offer a strictly autonomous solution. + +Discussing the reflection concept in detail is far beyond the scope of the examples collected here. We also talked about bitstream syntax and semantics using the example of XML files. We showed how to use the XML schema concept to describe details of the syntax and also the semantics of a document indirectly and to create the source code of a program that will be used in the serialization and deserialization process. + +Examples illustrating serialization using reflection and attributed programming are described later in the section [Reflection-Based Serialization Example][reflection-based-serialization-example] + +### Graph of Objects Serialization + +Let's move on to the last issue related to the serialization of objects interconnected to each other forming graphs. So the objects have references between them and these references will determine the structure of the graph of objects. In this case, the main challenge is that all the objects must be considered as one whole. + +Generally, we may distinguish two types of these structures. The first one is created using hierarchical interconnections, which resembles a tree. In this case, starting from any point of such a structure and following the directional references we never return to the starting point. Thanks to this feature, in mathematics this kind of graph is called acyclic. If graphs are cyclic then there are points in the graph that when we start from these points and follow the references, it is possible to return to the starting points. So such graphs have loops. + +Since graph serialization requires an iterative approach, it requires that we iteratively traverse a tree of objects, provided that it is a tree. If there are cyclic connections (causing loops in the graph of objects) then there is a problem with stopping the iteration and avoiding double serialization (cloning) of the same object. + +Assuming unidirectional interconnections between the objects and if a graph of objects is created as interconnected objects in such a way that they create a tree, or at least a layered model we can distinguish objects that are at the top of a hierarchy and objects that are beneath. Therefore, data transformation operations may be performed starting from those objects that are at the top and ending with those objects that are at the bottom of the hierarchy of references between objects. + +Unfortunately, often happens that we must deal with more demanding structures, where these references create cycles. For example, in this example (fig. below), classes refer to each other creating a cycle. + +![Fig. 1](../.Media/Part3-N80-10-Diagram.png) + +Assuming that instances of all classes are created (fig. below), the question arises which one of the objects in the graph should be subject to the serialization process first. Therefore, in this case, we must not insist that the hierarchy between objects is dependent on the order of representation in the stream. Hence, here we must introduce the following term, namely equivalence of streams. If a stream contains a representation of all information including references, the order in which the data associated with each instance is placed in the stream is not relevant, provided that each object is serialized only once. Due to the above, it has to be considered that several different bitstreams contain equivalent states of individual objects and these object states will be placed in different orders but all of them are equivalent to each other. It means that on their basis it will be possible to reconstruct an equivalent graph of objects. Creating equivalent streams does not mean that they have to be identical and therefore, for example, they can be directly compared with each other. + +![Fig. 2](../.Media/Part3-N80-20-Rekurencja.png) + +Another issue that should be addressed here is when the serialization process should be ended. For example, if we start with an instance of one class, let's say `ServiceA` (fig. above), next proceed to serialize the instance of the `ServiceB` class and consequently proceed to an instance of the `ServiceC` class, we must have an iteration stop condition to avoid cloning of the instance `ServiceA` because it has been already serialized, i.e. the transformation process has been performed for it. For the more complex graphs, it could be not so easy. + +In the case of cyclic graphs, there is no restriction on the number of paths between any pair of vertices, and cycles may be present. We may encounter two problems here. Firstly, we have to resolve many-to-one references in this type of graph, when many objects will have references to one object. As a result, we can expect that serializing such a structure may cause the cloning of objects in the stream. During recovery, if all these objects are recreated, many redundant copies are instantiated, so the structure will be different comparing it with the original. In the case of cyclic graphs (contain cycles - closed loops) in the relationship structure, we must take into account the fact that the serialization mechanism (the graph-to-bitstream conversion mechanism) will have to deal with this problem and therefore will have to set a stop condition to avoid cloning objects in the output stream. Well, we have two options to solve this issue. The first option is to write a custom library but this is a complex process. The second approach to address this problem is to choose an appropriate but existing library. There are many such libraries on the market and when analyzing their applicability, you should pay attention to these issues. + +## Self-controlled Serialization Example + +To illustrate this scenario, our task now is to implement a library class that enables reading from and assigning to properties defined as a member of a type that is a candidate to be serialized. + +The [SelfControlSerialization][SelfControlSerialization] class contains three properties. One of them is named `AverageIncome` and returns a calculated value. In other words, it returns the result of an expression executed using local values. This example shows that to recreate an object of this class, we only need to transform two values because the third one is always calculated, so there is no need to preserve it in the bitstream. The constructor of this class is responsible for recovering the initial values (state) of this class when the object is created. + +To transform an instance of this class (to serialize it), first, an attribute has to be associated with it that indicates that it is intended for serialization. However, this does not solve the issue of selecting the values contributing to the object state. The question is how to do it. + +The first approach to selecting values contributing to the object state is to have built-in functionality in the target type selecting appropriate values. It means moving the responsibility of state selection to the target class. Unfortunately, it solves only partially the problem because this functionality must be implemented each time a new type is defined. Let's look at the serialization process using a unit test. The test method [SelfControlSerializationTest][SelfControlSerializationTest] implements functionality well suited for this purpose. + +In this test class, we create an object of the target class that is to be serialized. In the next step, we must have a library class responsible for transferring a set of values to a bitstream. I have prepared a class called [CustomFormatter][CustomFormatter]. A bit later I will describe this class in more detail. After implementing the serialization functionality we can create a bitstream. Let's use a `FileStream` with a given name for this purpose and serialize the object to it, i.e. write a bitstream to this file content. After that, the content contains the state of the serialized object according to the selection made by the object itself thanks to the implementation of the [ISerializable][iserializable] interface. + +After preparing the serialization result, in the next step, let's check that this file exists and that its length indicates that values that constitute the object state have been written to it. Then let's read this file and save it locally to manually examine its content. + +But now let's move on to the implementation of the [CustomFormatter][CustomFormatter] class. Our [CustomFormatter][CustomFormatter] inherits from the [Formatter][Formatter] class. This class is defined in the language library and implements many operations to avoid cloning code and implementing the same functionality over and over. Using this type to implement serialization, it is assumed that the object to be serialized implements the [ISerializable][iserializable] interface. The main aim is to avoid using reflection and use embedded functionality that allows reading values that contribute to the object state. Thanks to implementing [ISerializable][iserializable], we can retrieve them from this object using the `GetObjectData` operation. + +Again, using this approach to implement serialization, it is assumed that the object type to be serialized implements the [ISerializable][iserializable] interface. The main aim is to read values that contribute to the object state using typical programming means. By design, this functionality must be provided by the target type. An example is in the [SelfControlSerialization][SelfControlSerialization] class. Thanks to this, we can retrieve the vital values from the target object by using the `GetObjectData` method. As a result of this operation, we have access to all values even private ones, and we can perform serialization operations for all values even invisible outside of the target object. So, we can repeat the writing of individual values thanks to the `WriteMember` method, which is implemented in the `Formatter` class. Next, there are operations related to creating an XML document and saving this document in a file. + +It is worth emphasizing that in our example we only write `double` values and it is a reason that only the `WriteDouble` method has been implemented. It creates an instance of the `XElement` type passing the value and key to the constructor. It is also important here that the [CustomFormatter][CustomFormatter] class is implemented in the library. + +To implement deserialization we must create an instance of the type and populate value holders to initialize the object state. To implement this responsibility, the dedicated [SelfControlSerialization][SelfControlSerializationConstructor] constructor must be provided. + +In this approach, the [CustomFormatter][CustomFormatter] class cannot have a reference to the definition of a target type that is subject to the serialization process. The object that is subject to the serialization process can be of any type but it must implement the [ISerializable][iserializable] interface and therefore must provide an implementation of the `GetObjectData` operation method. Because the target type is invisible, it can be recognized as a typical scenario where the dependency injection design pattern is required. + +## Reflection-Based Serialization Example + +It's time to move on to practical acquaintance with selected reflection mechanisms. To get more about reflection based on examples in selected programming language check out the document [Implementation Examples][ie]. These examples show how to represent type features as the [Type][system.type] class instances. The instances can be created using the `typeof` keyword or the `GetType` instance method for objects of unknown type. In both cases, an object-oriented type description is created. The examples discussed show how to use this description to read and write the values of a selected member of a type. This ability is especially useful when implementing serialization and deserialization operations. Similarly, we can also read and write values from fields and call instance methods. Similarly, it is also possible to create a new object without using the `new` keyword. + +This example explains how to serialize using reflection and attributed programming. Examples collected in this section are dedicated to demonstrating how to deal with the presented above scenario. It defines a few helper functions, for serialization and deserialization located in the static [XmlFile][XmlFile] class. + +The `ExDataManagement/DataStreams/DataStreams.UnitTest/Instrumentation` folder contains classes that represent the XML schema used by the program as an object model of the working data. An example of reflection-based data values access is the [Catalog][Catalog] class, which contains an array of CD descriptions. So here we have property, which is an array containing CD descriptions consistent with the [CatalogCD][CatalogCD] class defined in the same file. + +Classes to be serialized were defined in the test project. Therefore, if we define a library that will be used to serialize these classes, this graph, then the serializing class cannot know the type of serialized classes, cannot have references to unit tests, and so it cannot know the types. This way it could be proved that the solution is generic, I mean it doesn't depend on the definition of serialized classes. + +As we see in this example, we do not have to create custom code in the target type that is to be subject to serialization and used to implement this responsibility. So we can say that in this case, the serialization process is exactly autonomous. + +The main aim of the [SerializationUnitTest.ReadWRiteTest][ReadWRiteTest] method is to test the serialization of the graph of objects represented by the [Catalog][Catalog] class. To be tested, the instances must be populated with test data. This class is located in unit tests, so I can add an appropriate method that populates the instance of this class with test data. The only question is where to add it. Adding this method to auto-generated text, i.e., text obtained as a result of an external program, is not a good idea, because our work is overwritten after each modification and generation of a new text. Therefore, let's take advantage of the fact that this class is generated as a partial class, and to populate the instance of this class with test data, we have to expand its definition by adding a custom part, which will be its integral part. In this part of the definition, located in a separate file, we can safely add all the operations we want to perform for this purpose. For this method called `AddTestingData` I used attributed programming again by adding an attribute that indicates that this method will be subject to compilation only when we have an environment configuration named [DEBUG][Debug]. + +Coming back to the unit tests, we see that an object has been created, and this object has been populated with test data. To make sure, we check that the instance has been created and initialized. Then we define the path where we want to save the file and use the `WriteXMLFile` method. This is a generic method. In its first parameter, we pass a graph of objects to be serialized, the file name, and details related to the output file creation. + +The [WriteXmlFile][WriteXmlFile] method has been defined in the library. We will not analyze it in detail but the only important thing is that we use the `XMLSerializer` library to perform the serialization operation. The serialization is implemented in this statement: + +``` csharp + _xmlSerializer.Serialize(_writer, dataObject); +``` + +By design, all other instructions are used to protect against wrong values of parameters and to improve the formatting of the output XML text. + +For testing purposes, an operation is performed to read the same file and create an equivalent graph of objects, i.e. deserialization implemented in the following assignment instruction + +``` csharp + Catalog _recoveredCatalog = XmlFile.ReadXmlFile(_fileName); +``` + +We can now check whether the result is consistent with our expectation, i.e. whether the original graph of objects and the equivalent graph of objects have appropriate values that are part of the object's state. + +There are two more things worth noting about the [ReadWRiteTest][ReadWRiteTest] method. The first one is reading the stream and restoring an equivalent graph of objects. The second one is to check whether the graph of objects is equivalent compared with the original one. As we can see in this method, the same library called `XMLSerializer` is used. As previously the operation of restoring the graph of objects comes down to one instruction. From the example we can derive that testing if the recovered graph of objects is equivalent to the original one strongly depends on the custom type definitions and cannot be performed universally, therefore it must be the responsibility of developers. + +Thanks to the presented example we may learn how attributed programming and reflection may be used to ensure autonomy and synchronization of object-to-stream conversion processes and vice versa. In the presented example there is no need to point out members contributing to the object state. In the [Catalog][CatalogAttributes] class attributes are only used to control the behavior of the serialization process. + +Although we know that this is not a universal approach, let us return to the discussion of the topics related to checking the equivalence of the recovered graph compared to the original graph in this specific case. The primary graph was created while creating an object of the [Catalog][Catalog] class and then filling it with test data using the `AddTestingData` method. After deserialization, we check that the `_recoveredCatalog` variable has references to the newly created object, so it is not `null`. Then we check how many elements the array has. It is assumed that there are only two elements, but it would also be worth checking the actual length of the array. However, the most important thing here is to check whether two subsequent disc descriptions compatible with [CatalogCD][CatalogCD] are equivalent to each other. The equality symbol is used to compare them, although we expect that the elements are equivalent, not identical. This effect can be achieved by redefining the equality operator in the [CatalogCD][CatalogCD] class. For this purpose, the definition of the equality operator has been overwritten. As a result, the behavior of a new definition of this operator determines what equals means. The standard `Equals` method is used here. This operation compares strings, which have been generated by the overridden `ToString` method. It determines which elements will take part in this comparison and how they will be formatted. It is worth emphasizing here that the string formatting may depend on the current operating system language settings and, depending on different data types, the formatting of this string may not be clear; it may not be the same every time. + +## See Also + +- [References](./../../../REFERENCES.md#references) + +[ie]: README.md#implementation-examples- +[self-controlled-serialization-example]: READMESerialization.md#self-controlled-serialization-example +[reflection-based-serialization-example]: READMESerialization.md#reflection-based-serialization-example + +[system.type]: https://learn.microsoft.com/dotnet/api/system.type +[Debug]: https://learn.microsoft.com/visualstudio/debugger/how-to-set-debug-and-release-configurations +[Formatter]: https://learn.microsoft.com/dotnet/api/system.runtime.serialization.formatter +[iserializable]: https://learn.microsoft.com/dotnet/api/system.runtime.serialization.iserializable +[WriteXmlFile]: Serialization/XmlFile.cs#L41-L62 +[CustomFormatter]: Serialization/CustomFormatter.cs#L21-L153 +[XmlFile]: Serialization/XmlFile.cs#L22-L97 + +[ReadWRiteTest]: ../DataStreams.UnitTest/SerializationUnitTest.cs#L42-L58 +[Catalog]: ../DataStreams.UnitTest/Instrumentation/Catalog.xsd.cs#L21-L55 +[CatalogAttributes]: ../DataStreams.UnitTest/Instrumentation/Catalog.cs#L19-L25 +[CatalogCD]: ../DataStreams.UnitTest/Instrumentation/Catalog.xsd.cs#L56-L79 +[SelfControlSerialization]: ../DataStreams.UnitTest/Instrumentation/SelfControlSerialization.cs#L22-L79 +[SelfControlSerializationTest]: ../DataStreams.UnitTest/SerializationUnitTest.cs#L26-L39 +[SelfControlSerializationConstructor]: ../DataStreams.UnitTest/Instrumentation/SelfControlSerialization.cs#L61-L66 diff --git a/ExDataManagement/DataStreams/README.md b/ExDataManagement/DataStreams/README.md index 744ed048..aa28b715 100644 --- a/ExDataManagement/DataStreams/README.md +++ b/ExDataManagement/DataStreams/README.md @@ -1,66 +1,287 @@ -# Section 2 - Data streams + -## Key words +# Data Streams -File System, User Interface, XML, XSLT, HTML, XmlSerializer, Save file, Transformation, Saving text files, Local File Systems, Open and read file, XML Schema, Common File Format, Data Access, XML Serialization, XSL, Data Validation, XML Documentation +## Introduction -## Lesson 03 - File and Stream Concepts +To use computers to automate information processing we have to deal with bitstreams as the information representation. By design, bitstream management involves the organization, storage, retrieval, communication, and manipulation to ensure accuracy, security, and accessibility. It encompasses data collection, storage architecture, integration, and maintenance to support efficient analysis and decision-making. To fulfill these requirements, the following concepts could be a real relief. It includes but is not limited to presentation, validation, standardization, serialization, and safeguarding of data. -## Lesson 04 - Attributes +This folder `ExDataManagement\DataStreams` contains examples related to information representation as a bitstream and is devoted to discussing selected programming issues related to their management. -## Lesson 05 - Reflection +## File and Stream Concepts Preface -## Lesson 06 - Serialization +If we write a program to automate information processing, we inevitably have to operate on data representing this process. Generally, we can distinguish operations related to reading input data, permanently preserving intermediate data, transferring data between individual applications, and saving the final data somewhere after completing the entire processing process. All these requirements can be accomplished using the concept of file. Even sending data between applications can be done using a file server, distributed file system, Google Drive, One Drive, and Pendrive to name only the most popular ones. -## Executive summary +This is where the term file system came into play. Without going into details about the architecture of the computer and the operating system, we can enigmatically state that the file system is a resource available in virtually every modern computer. For us, its most important feature is the ability to manage files. First of all, the file is metadata, i.e. data describing data. So here we may have the first indication that we are talking about data that we are describing. One such description is an identifier that plays two roles. One is that it clearly distinguishes it from all other files. In this role, it could be recognized as a Uniform Resource Identifier(URI) - the text that is a unique identifier of a file among others. The second one indicates the location, namely where the file may be found by the file system engine. In this role, it is a Uniform Resource Locator (URL). We also have other metadata such as date of creation, author, length, and many others. -This sample demonstrates how to save working data in a file containing xml document, which next can be directly presented for example in MS Word editor or Internet Explorer translated using a style sheet. It is simplest way to detach the document content from its presentation. +An important feature of a file concept is that it contains content in addition to metadata. Metadata is, of course, a very important component of any file but the most important thing is the content it includes, which is data representing information to take part in processing. -## Scenario +Hopefully, everything we've talked about so far seems quite obvious, but since some file features are fundamental for further discussion, let's look at them in detail. -Applications save working data into the files to keep state information, to provide processing outcome or both. Applications need robust storage, i.e. correctness of the stored data has to be validated every time an application reads it back from the file. It must be carefully observed if the files are also modified by other applications or directly by users, because data corruption may occur. To address the validation requirement XML (Extensible Markup Language) as a text-based format for representing structured information and XML Schema as a language for expressing constraints about XML documents are very good candidates to be used by the save operation. +Let's start with the fact that typically we utilize object-oriented programming. This means that at design-time we have to deal with reference types and at runtime, we must deal with objects located in a computer's working memory (RAM). Let me remind you that the RAM abbreviation stands for Random Access Memory. Here, random means that each word in memory has an address, i.e. a unique identifier, and this word can be independently read or written there. Let me stress we are talking about freedom but not probability. It means that again the RAM address plays the role of URL. Therefore, object data can be organized into structures and linked by references. -## Presenting working data outside the application +On the other hand, we have the streaming world where the data is organized in the form of bitstreams, where each bit of data has only information about the next one. -As the XML format is text based it can be directly read and displayed by the software user. However, it is not preferred format, because it does not contain any formatting information. Today we expect data presentation to meet user experience, i.e. to have appropriate layout and style. We can meet this requirement using any application that supports XSLT transformation of XML documents into other text documents or HTML documents. XSLT uses a template-driven approach to transformations: you write a template that shows what happens to any given input element. For example, if you were formatting a working data to produce HTML for the Web, you might have a template (*stylesheet file*) to match an underlined group of elements and make it come out as a table. +## Useful Concepts for Bitstreams Deployment -To get more about how to start with XSLT visit the W3C School: +### Introduction -[XSL\(T\) Languages](https://www.w3schools.com/xml/xsl_languages.asp) +To use computers for automation of information processing we have to recognize bitstreams as the information representation. By design, bitstream management involves the organization, storage, retrieval, communication, and manipulation to ensure its accuracy, security, and accessibility. It encompasses data collection, storage architecture, integration, and maintenance to support efficient analysis and decision-making. To fulfill this functionality the following set of concepts could make a real relief. It includes but is not limited to presentation, validation, standardization, and serialization operations. -Today applications use objects to process working data according to the Object Oriented Programming (OOP) paradigm. To save/read working data from XML files we need generic operations that could automate this process regardless of types we used to create the object containing working data. This process is called serialization. There must be also provided reverse operation creating objects from the XML content-deserialization. This operation additionally has to verify the content against the XML schema. Instead of XML schema to validate the XML file validation we can use equivalent set of classes. +### Presentation -To learn more about the serialization visit the MSDN: +Bitstreams presentation is implemented by various ways of conveying information, including textual and tabular formats. Hence, first of all, we need to deal with data presentation, so as to enable the use of bitstreams also by a human computer user. In this context we must take into account the following terms: natural language, ergonomics, and graphical user interface. -[Serialization in .NET](http://msdn.microsoft.com/library/7ay27kt9.aspx) +A typical example that we can cite here is using the Internet. Using a web browser, a server-side application uses objects and then serializes the data that the user needs, sends it over the network, and then the browser displays it on the screen. And here it is important, that the browser always displays data in graphical form. This applies to all kinds of data used by humans. Data, even when reading a newspaper, is always formatted as a graphical presentation. Let me remind you that any character is also a small picture. This is one feature of data that is prepared for this. so that man can use them. The second feature is that this data must be written in a natural language that humans know. The concept of natural language is very broad. For example, XML text is said to be human-readable. But is this a piece of natural language? -You may use the XML Schema Definition [Xsd.exe](http://msdn.microsoft.com/library/x6c1kb0s.aspx) tool, which generates XML schema or common language run-time classes from XDR, XML, and XSD files, or from classes in a run-time assembly. +From the above, we can derive that the bitstream should be formatted to resemble a natural language. Of course, we have no measure here and therefore it is difficult to say whether something is close enough to natural language to be comprehensible. -## Solution +### Validation -This example is dedicated to demonstrate how to deal with the presented above scenario. It defines a few helper functions, for serialization and deserialization located in the static class: +Applications save working data into bitstreams (for example content of files) to keep state information, provide processing outcomes, or both. Applications need robust storage, i.e. correctness of the stored data has to be validated every time an application reads it back from a bitstream. It must be carefully observed if the bitstreams are also modified by other applications or directly by users, because data corruption may occur. -`Example.Xml.DocumentsFactory.XmlFile` +If we are talking about exchanging data between different applications or between an application and a human, the issue of data correctness arises. This issue should be considered on two independent levels. The first one is the correctness of a stream of signs, i.e. validation if the selected syntax rules are met. The second one is the possibility of assigning information (meaning) to these correct sequences and therefore assigning meaning to a bitstream. For humans to understand the stream, it will be accomplished by defining semantics rules, i.e. rules that will allow us to associate meaning with bitstream. The issue of ergonomics is also important in how easy it is to absorb information represented by the bitstream. Of course, the closer we are to natural language, the easier it will be, but again in this matter, we do not have measures that will allow us to determine how good our solution is. -The `Example.Xml.CustomData` namespace contains classes that represent XML schema and are used by the program as an object model of the working data. +To better understand the above mentioned topics, let's look at them in the context of code examples explained in the section [XML-based Validation][xml-based-validation]. In this section, XML examples are only subject to more detailed examination but by design, it has no impact on the generality of the discussion. -After implementation of the `Example.Xml.DocumentsFactory.IStylesheetNameProvider` by the root class of the object model we can convey information about default stylesheet that may be used to transform resultant XML file. Information about the stylesheet (xslt file) is added to the XML document and can be used by the application to open the file and translate the content. +### Standardization -An example of xslt file has been added to the CustomData and is copied during project build to destination folder. In the same folder an example of XML file (named Catalog.xml ) is created. You can open it using IE or MS Word using the instruction below. +When we talk about the syntax and semantics of a stream, the first thing to consider is the scope of data use. Data produced by one program instance can also be used by the same program instance. In such a case, if the process runs autonomously and is symmetric from a serialization and deserialization point of view, we should not expect any further problems. -Program class demonstrates how to use read/write operation. +If we are talking about interoperability between different applications, we must consider a situation in which these applications have been written using different programming languages. In this case, the problem arises of how to create types in other languages that will represent the same information. In the context of a text document, a kind of schema may be used. -## Lesson 07 - Cryptography basics +The schema in this context refers to a bitstream structure or blueprint that defines the organization and format of the document. It outlines the arrangement of elements, their relationships, and any rules or constraints that govern the content of documents. Simplifying, schema allows the definition of additional syntax rules in a domain-specific language. Schemas help ensure consistency in the representation of information within the text document. It means schema definition could also be a foundation of semantics rules that assign meaning to the document text. As a result, we could recognize the schema as a good idea to validate text documents and check whether incoming text is a document we expect. Instead of using a schema to validate text-based bitstreams, we may use an equivalent set of classes. - +Because the data may be used by different instances of a program, we also have to take into consideration that the applications may be in different versions or written using different languages. What is worse, the data representation also must be subject to versioning. In such a case, there is a problem of data compatibility between independent instances of the program. So the question arises whether the data serialized by one version of the program is used by another version of the program run as a different instance. + +Another very popular applicability domain of streams may be the use of them to implement interoperability between instances of various programs that are created using different technologies and implemented on different platforms. Then there is also the issue of technological compatibility. Also in this case, it must be taken into consideration that classes (types) that were created in one technology cannot necessarily be directly used in another technology. In this case, we must take into account that in another technology the same information will be represented differently. + +If schema definition is expressed in a widely accepted format it should be possible to generate types in selected programming language based on this schema. Of course, it is a chicken and egg problem namely, should we first create types in the selected programming language, or should we create these types in the schema and then create classes based on the schema definition? But let's try to see how this can be achieved using an example. + +### Serialization + +We need bitstreams to be handled using files to make sure that the data can be persisted. Let's recall the most important applications, such as entering input data or storing output data using file systems. We also use various types of streaming devices to archive data, i.e. to save data forever. The temporary and intermediate data repository is another example. Data transfer between applications is another use case. It requires that data must be transferable. For example interoperability of a web server and a web browser. There is a virtual wire between them. The virtual wire is not an abstract interconnection but means that only bitstream can be transferred between them. There are many more examples but let's limit the discussion to the mentioned only because they are enough to justify the importance of this topic. + +In the already mentioned use cases, data must be in the form of bitstream. Now we are ready to return to discussing issues directly related to streaming data. Previously, we discussed the mechanisms of managing streams, especially in the context of files. We also realized the differences between bitstreams, text, and documents. Now let's answer the question of how to create streaming data and how to use it. First, let's try to define the purpose of our missions and the limitations we must deal with. + +The first problem is related to the inevitable necessity of dealing with two concepts, namely object data with the data formatted as bitstreams. The transition process from the objects to the stream is called the serialization. Deserialization is the reverse process, which involves replacing the bitstream with interconnected objects located in the working memory of a computer. Hence, in the context of serialization, to save working data in a file we need a generic operation that could automate this transition process regardless of the types we used to create the graph of objects wrapping working data. There must be also a reverse operation creating objects from a file content - deserialization. To guarantee consistency, this operation has to verify the file content against the types used to instantiate objects. + +Again, in the transition between the world of objects and the world of bitstreams, we need serialization, which is responsible for the transition of the state of a graph of objects to a bitstream. And deserialization, which is responsible for the reverse process, i.e. for transferring a bitstream into a graph of interconnected objects. We would like these operations to be implemented as generic, i.e. we would not have to program these operations every time, but only parameterize it. + +Before we move to the next step, it is worth recognizing what we need to implement this functionality. Here, from the world of objects point of view, the list of requirements includes: + +- access to the values wrapped by objects that will be the subject of the serialization - in other words, values that will constitute the state of the objects +- the relationships between these objects + +Next, we need to implement an algorithm that will describe in detail this data transformation, which has to be mutually unambiguous. Here, the mutual unambiguity of this process does not mean that each time we perform serialization we will obtain an identical bitstream. The same we should state for deserialization. We will get back to this issue shortly. + +So the first problem we have is how to implement serialization and deserialization to make the transition between the object world and the streaming world possible. The serialization and deserialization process must be mutually unambiguous operation. Moreover, it is not a simple process. Well, someone may say that this is a relative matter because we have no firm metrics of simplicity in this case. However cloning serialization and deserialization code snippets each time serialization is needed will consume and waste time, so it may be worth implementing this process as a generic library, without the need to create dedicated software each time. So the next problem we can define here is the possibility of transition between the streaming world and the object world using the library concept. + +If we talk about repeatability by applying a library concept implementing serialization and deserialization functionalities, we need to offer a generic implementation. Namely, we must be able to define this process in advance, without prior knowledge of what will be serialized. Generic implementation of the serialization and deserialization functionality means that we have to implement it in advance and offer it as ready-to-use libraries. + +Today on the market, we have many libraries that permit this process to be carried out automatically. So it is justified to ask the following question why do we need to learn about it? Why go into detail? Well, my point is that if someone wants to use a washing machine, let me refer to this example, they do not need to know how the controller, engine, or temperature sensor works. However, if someone wants to assemble a custom washing machine using available parts, knowledge or understanding of how the engine, controller, and temperature sensor work is essential in this case even if the mentioned parts are available. Similarly, we need detailed knowledge about how to manage bitstreams in case we are going to use streaming data, for example, the file system. + +In summary, to simultaneously use data as objects and bitstreams, our goal must be to combine two worlds. First, in which the data is in object form. The second world contains data in the form of bitstreams. Let me stress now that in both cases we have the same information but different representations. The data conversion between these worlds is called serialization and deserialization. In the case of serialization, it is a process that involves converting the state of a graph of objects into a bitstream. Deserialization is the reverse process, i.e. converting a bitstream into a graph of objects that must be created in memory. Here the magical statement about the condition of the object appeared; what does object state mean? We will learn the answer to this question soon. + +From the above, it could be derived that if an equivalent graph of objects can be reconstructed based on a bitstream it can be stated that the bitstream is correct for the purpose it has to serve. This reconstruction must be accomplished in compliance with the syntax, and semantics rules governing the bitstream. Again, this graph does not have to be identical to the original each time. It is enough for us that it is equivalent from the point of view of the information it represents. It could be added that in some cases, let's say in simpler cases, the bitstream identity can be ensured. This means that for a selected graph of objects, each time as a result of serialization we receive an identical bitstream. Then this bitstream can be compared, for example, to check whether the process is the same as before. It must be stressed that equivalence has no metrics measure that can be applied to evaluate equivalence conditions. Due to the above, it is not possible to formally determine whether the resulting bit stream and the source object graph are equivalent. Therefore, equivalence must be decided by the software developer using custom measures, for example, unit tests. From that, we can derive that only the software developer is responsible for ensuring that serialization and deserialization are mutually unambiguous. + +Assuming that the data transformation algorithm has been implemented somehow, there is a need to determine the format of the target bitstream. So we need to determine how to concatenate bits into words, words into correct sequences of words, and how to assign meaning to these sequences of words. Shortly, a set of valid characters, syntax, and semantics rules are required. For example, it could have an impact on the bitstream features, like the possibility of validating and visualizing content using existing tools. Two additional notes regarding the target format of the bitstreams are vital for further consideration. + +The list of applications - mentioned previously as potential bitstream consumers - includes the exchange of data between remote applications. It should be emphasized here that if these applications are created by different manufacturers, the standardization of this representation becomes extremely important. So, the fact that we combine words into correct sequences of words and assign to them meaning, that these syntax and semantics rules are standard in the sense that there are international documents that are published by organizations recognized as standardizing, that will allow us to recreate the graph of objects in applications that are created by other vendors. + +We also said earlier that sometimes these bitstreams are also used to communicate with humans. Of course, standardization is also important for this kind of application. A bitstream user must be able to read this sequence of bits, and therefore combine sequences of bits into words and words into correct sequences of words. Finally, these strings of words have to have meaning for him. First, it is important to be able to apply encoding to create characters so that the bitstream becomes a text. Let me remind you that the text is a bitstream for which encoding is known in advance or discoverable somehow. + +From the previous considerations regarding the transformation of object data into streaming data, we know that the basis of this process is to determine the state of the object. Let me remind you that the state of an object is a set of values that must be subject to a transformation process so that the reverse operation can be performed in the future, i.e., so that the object graph can be recreated and an equivalent object graph can be created. + +In order not to enter into purely theoretical considerations, let us return to these topics in the context of sample programs. The examples are described in the document titled [Objects Serialization][objects-serialization]. The example discussed shows the mechanism of transformation of an object or more precisely an object state to a bitstream. In this process, the state of the object is determined by a software developer, which implements an appropriate mechanism responsible for selecting the values that constitute the object state. Since the determination of an object state is the responsibility of program authors, there must be measures allowing them to point out what has to be serialized. + +To implement a serialization/deserialization engine, you need to define a data structure, choose a serialization format (like custom, JSON, XML, etc.), and use a serialization library to convert the data wrapped by a graph of objects into the selected format in both directions. The data structure is required to determine the state of objects that are subject to serialization. Apart from the data structure a guidelines allowing to select only values constituting the state of the object are necessary. To fulfill the mentioned requirements access to the value holders that constitute the state of the object is also required. Attributes as a language construct at design-time and reflection as a technology at run-time could help to solve some problems related to serialization/deserialization implementation. + +> To learn more about the serialization in .NET, visit the document: [Serialization in .NET][STLZTN]. + +### Cybersecurity + +#### Introduction + +Cybersecurity describes the practice of protecting computer systems, networks, and data from cyber threats. In this section, cybersecurity related to bitstreams is considered. Now let's talk about securing streams using cryptography. Talking about cryptography in the context of streams may seem a little strange because usually cryptography is discussed in the context of data security and system security in general. Cryptography is a broad concept, but we will focus only on selected, very practical aspects related to the security of bitstreams. + +We already know how to create bitstreams. We can also attach coding to them, i.e. the natural language set of characters. The next step is to assign syntax and semantics that allow the streams to be transformed into a coherent document, enabling a computer user to recover information from these documents. If this is not enough, we can also display these documents in graphical form. + +It must be stressed again that in all occurrences this computation infrastructure is always binary, and we must consider that it may be sent over a network, archived, and processed by another computer. Hence, it is required that the bitstreams are protected against malicious operations. For example, if a document contains a wire transfer order to our bank, the problem becomes real, material, and meaningful in this context. + +In the context of the cybersecurity of bitstreams implementation, the following requirements must be encountered: + +1. ensure that all users of a source bitstream can verify that the stream has not been modified while it was being archived or transmitted, +1. safeguard information from unauthorized access, ensuring confidentiality, +1. confirm authorship, so all users of a bitstream can determine who created it and who is responsible for its content. This requirement we call non-repudiation of the author. + +A short description of methods used to protect a bitstream against malicious users may be found in the following chapters. Examples illustrating how to implement them are collected in a separate document [Bitstream Cybersecurity][READMECryptography]. + +#### Hash + +If we are talking about archiving streams or transferring streams from one system to another, from one computer to another, the first thing we need to take care of is the integrity of such a stream. This means that from the moment it is produced until it is at its actual destination, where it will be processed, it is not modified. The best way to accomplish this is by using the hash function. + +Thanks to the hash function, we can secure the integrity of the controlled bitstream, provided that we can transfer the hash function value to the destination in such a way that malicious users cannot modify it. Otherwise, modifying the source stream is not a problem because calculating a new hash function value that takes this modification into account is quite a trivial operation. + +Bitstream integrity refers to the assurance that the bitstream remains intact during transmission or storage. It ensures that each bit in the data stream retains its original value without corruption or errors. + +#### Encryption + +It often happens that only authorized persons should have access to the information represented by a bitstream. To address this requirement, we may use the bidirectional transformation mechanism to replace a source bitstream with another bitstream to which we can no longer attach the encoding, syntax, and semantics rules. As a result, it makes it impossible to associate information with this bitstream. The bitstream is no longer meaningful data. The obtained from the transformation bitstream resembles white noise. However, any person who has the right to access the associated with the source bitstream information; should be able to recover the source bitstream and as a result associate back the encoding, syntax, and semantics rules. As a result, it allows recovering the information represented by the source bitstream. The process is similar to replacing music with noise but granting the possibility to recover music from that notice by the authorized user. Unauthorized users can hear only noise, but authorized users can transform the noise back to the original music. This reversible transformation function we will call encryption. + +Hence, selective access is required to protect any bitstream including but not limited to hash value against unauthorized access. Selective access is the ability to access information that is associated with a bitstream only by people who are authorized to do so. We can accomplish this in two ways: + +1. selective availability of the bitstream itself +1. selective availability of the bitstream meaning + +The first approach is to share the bitstream, for example, as a file, only with people who have the right to get access to it. This can be achieved thanks to the authentication and authorization offered by most operating systems. Authorization in the context of an operating system refers to the process of granting or denying permissions to identity attempting to perform certain operations on a computer system. Thanks to this, each time an attempt is made to operate on a file, it is first checked whether the identity that requested the execution of an operation has the right to do so. Of course, if someone does not gain access to the bitstream (to the file content), he will necessarily not have access to the information that is associated with this bitstream. Unfortunately, this approach is possible only in case there is something trusted in the middle between the file and the user, for example, a well-configured operating system. This topic generally doesn't deal with operating systems implementation, so this approach is outside the scope of our interest. Hence, we have to deal with another security method. + +The second option is to transform a bitstream (for example the file content, hash function value, etc.) into a form that an unauthorized user cannot associate any information with this bitstream. This method we call encryption. In other words, encryption involves transforming or scrambling bitstreams to make the underlying information unavailable to unauthorized users. + +Bitstream encryption encompasses encrypting data at the bit level, which is often used in various scenarios. These scenarios employ encryption techniques to protect data integrity, confidentiality, and privacy against unauthorized access and interception. We can distinguish between symmetric and asymmetric encryption. Symmetric encryption uses the same key for encryption and decryption, while asymmetric encryption uses a pair of keys: a public and a private key. + +We have already learned about the hash function to protect bitstream integrity. However, there is still a problem with how to distribute its result so that in different places of the IT system, and different locations in the world this hash value can be used to check the integrity of a bitstream. In the case of symmetric encryption, in which we use identical keys for the encryption and decryption, inter-operating parties have the same problem of distributing these keys among the authorized users who have the right to access the information represented by this stream. There is another problem with the use of symmetric encryption, namely scalability. It consists of the fact that the number of keys that we need to manage for encryption and decryption increases rapidly, that is, it increases with the square of the number of parties that participate in the data sharing. + +However, the main drawbacks of asymmetric encryption include: + +1. **Computational Overhead** - asymmetric encryption algorithms are typically more computationally intensive compared to symmetric encryption, requiring more processing power and time for encryption and decryption operations +1. **Limited Performance for Large Data** - asymmetric encryption is less efficient for encrypting large amounts of data compared to symmetric encryption, making it less suitable for bulk data encryption. + +Overall, while asymmetric encryption provides valuable security features such as key distribution and digital signatures, it also introduces complexity and performance limitations that must be carefully considered in design and implementation. A tradeoff is needed to deploy the cryptography. Usually, a temporary key is generated, and before use protected by asymmetric encryption. This way only a symmetric key is protected instead of the target bitstream. + +#### Non-repudiation + +When talking about documents such as a wire transfer order, there is no need to provide any special justification that the recipient of such a document will be vitally interested in being able to determine that the document has been issued by an authorized person, for example by the owner of the account for which the order was issued. + +Because we use file systems and transfer bitstream data over computer networks non-repudiation of streaming data inherently must be the subject of our particular concern. Non-repudiation can be achieved by providing a way to verify that the sender of a bitstream is who claims to be and that the bitstream has not been altered during transmission. To achieve this protection a digital signature is applied. The digital signature is a cryptographic technique used to ensure the authenticity and integrity of a bitstream. + +To implement a digital signature, the sender uses a private key to create a unique digital signature for the bitstream. This private key is known only to the sender and is kept confidential. The recipient, in turn, can verify the signature using the sender's public key. The public key is widely distributed and can be freely shared. + +If the digital signature is valid, it confirms that the bitstream is indeed signed by the holder of the private key associated with the public key used for verification. The digital signature also ensures that the content of the bitstream has not been altered since the signature was created. Even a bit of change in the bitstream causes a completely different signature. + +## BitStream Format + +### Domain Specific Language (DSL) + +Using bitstreams (file content) we must look out for a problem with how to make bitstreams human readable. The first answer is that it must be compliant and coupled with a well-known application. The application opens this bitstream as input data and exposes it to the user employing appropriate means to make the data comprehensible. + +Unfortunately, this approach does not apply to custom data. Therefore we should consider another approach, namely human-readable representation should be close to natural language. The first requirement for humans to understand the stream is that it has to be formatted as text. To recognize bitstream as the text an encoding must be associated by default, directly or indirectly. The next requirement, common for both humans and computers, is that a bitstream must be associated with comprehensible syntax rules. Finally, semantics rules should be associated with the bitstream that allows to assigning of meaning to bitstreams. Shortly there have to be defined a text-based language. A domain-specific language (DSL) is a text-based language dedicated to expressing concepts and data within a specific area. Except for programming languages like Java, C#, and Python, examples of well-known and widely accepted domain-specific languages are XML, JSON, and YAML to name only the most crucial. + +Using DSL to describe the bitstreams a Data Transfer Object (DTO) concept can be used as a foundation to encapsulate and transport data between computer programs. It may be a text document that contains fields to store data. + +To use DTO in a multi-vendor environment to transfer data between instances of different programs the standardization of the syntax and semantics rules is vital. Additionally possibility to use well defined and widely accepted schema documents is a key feature to establish interoperability. + +### Extensible Markup Language (XML) Format + +#### Introduction + +Extensible Markup Language (XML) is a standard text-based format for representing structured data in machine-readable form. Because it is founded based on the text it could also be recognized as human-readable. Its simplicity and flexibility make it suitable for representing a wide range of data formats. + +It consists of markup tags that define elements within a document. Each element can have attributes and contain nested elements, forming a hierarchical structure. The basic syntax involves opening and closing tags to encapsulate data. Attributes provide additional data in context of the opening tag. + +XML is often used for data interchange between different applications. + +Overall, XML is versatile and widely adopted in various domains for configuring settings and exchanging process data. + +#### Visualization + +As the XML format is text-based it can be directly read and displayed by a variety of software tools. However, it is not the preferred format, because it does not contain any formatting information. Today we expect data presentation to meet user experience, i.e. to have an appropriate layout and style. We can meet this requirement using any application that supports XSLT transformation of XML documents into other text documents, including but not limited to equivalent HTML documents. XSLT uses a template-driven approach to transformations: you write a template that shows what happens to any given input element. For example, if you were formatting working data to produce HTML for the Web, you might have a template (stylesheet file) to match an underlined group of elements and make it come out as a table. + +Let's go back to the the question of how to visualize data for a user, for a human. It was stated that an XML file is text, namely a bitstream for which the encoding is defined. It allows to employ of any text editor. Unfortunately, if a file is formatted this way and is seen by persons, who are not familiar with XML technology, it won't be easy to associate any information with the text. In this context reading the document and understanding the document are not the same. + +To make it easier to visualize the data that is in the XML file, let's use a feature of XML files that allows a transformation of XML text to any other text. Finally, a few notes related to XML stylesheet transformation. Not only web browsers have a built-in mechanism ensuring transformation. This transformation can be defined in such a way that the target text that will be created has the features of a natural language. The final form may also cover ergonomic requirements, and in particular, it may be the user interface. Shortly, thanks to the transformation of XML files using stylesheet it is possible to add formatting to the data contained in the XML bitstream. + +> - To get more about how to start with XSLT visit the W3C School: [XSL(T) Languages][XSLW3C] +> - To check out an examples visit the section [XML-based Presentation][xmlpresentation] + +#### Validation + +To address the validation requirement XML (Extensible Markup Language) as a text-based format for representing structured information and XML Schema as a language for expressing constraints about XML documents are very good candidates to be used by the file operation. Today applications use objects to process working data according to the Object Oriented Programming (OOP) paradigm. + +You may use the [XML Schema Definition Tool (Xsd.exe)][XSD], which generates XML schema or selected language classes from XDR, XML, and XSD documents, or from classes in a run-time assembly. + +To better understand topics related to validation check out code examples described in the section [XML-based Validation][xml-based-validation]. + +#### Standardization + +Extensible Markup Language (XML), is a standardized markup language designed to store and transport data. It provides a set of rules for encoding documents in a machine-readable format. XML standardization ensures consistency in data representation and interchange across different systems. + +Visit the `See also` section to get more details. + +### JavaScript Object Notation (JSON) + +#### Introduction + +JavaScript Object Notation (JSON), is a lightweight data interchange format. It is a text-based domain-specific language that is easy for humans to read and write, and for machines to parse and generate. JSON is often used to transmit data between a server and a web application, as well as for configuration files. It consists of key-value pairs and supports data types like strings, numbers, objects, arrays, booleans, and null. + +#### Visualization + +Yes, JSON can be transformed into other text formats using a variety of programming languages employing additional libraries for parsing and then converting to different formats like CSV, XML, or others as needed. Languages like JavaScript can be also used for transforming JSON documents to other text formats. JavaScript has built-in functions for JSON manipulation, and you can use libraries or frameworks to convert JSON to various formats as needed. + +#### Validation + +Thanks to schema definition it is possible to derive new domain-specific languages based on JSON. + +To address the validation requirement JSON (JavaScript Object Notation) as a text-based format for representing structured information and JSON Schema as a language for expressing constraints about JSON documents are very good candidates to be used by the operation on bitstreams. + +You may use a lot of available in the open access domain tools, which generates XML schema or selected language classes from different kinds of documents. + +To better understand topics related to validation check out code examples related to XML described in the section [XML-based Validation][xml-based-validation]. XML is used to express a general disunion using concrete language. + +#### Standardization + +This language is recognized as an international standard. It is standardized by the International Organization for Standardization (ISO) as [ISO/IEC 21778:2017][ISOJSON]. The standardization ensures that JSON is consistent and widely accepted for data interchange between different systems and programming languages. There is also [Request for Comments:7159][RFCJSON] specification titled _The JavaScript Object Notation (JSON) Data Interchange Format_. + +ISO/IEC 21778:2017 specifies the JSON data interchange format, its data model, and its various data types. JSON's simplicity, ease of use, and language-agnostic nature have contributed to its widespread adoption in various domains for representing and exchanging data. JSON is also supported by an open community maintaining schema specification [JSON Schema][CommunityJSON] + +### Yet Another Markup Language (YAML) + +#### Introduction + +YAML, short for "YAML Ain't Markup Language" is a human-readable data serialization format. It is often used for configuration files and data exchange between development environments with different data structures. YAML uses indentation to represent hierarchy and relies on a straightforward syntax with key-value pairs. It aims to be easy to read and write, making it popular in various applications, including configuration files for software projects. + +#### Visualization + +YAML doesn't define any special language allowing automatic transformation of YAML document to other text-based documents that can be used to visualize associated information. To visualize the content of a YAML document, you can use various tools and editors that support YAML. Here are a few options: + +- **Online YAML Editors**: Use online YAML editors like YAML Online Viewer or YAML Lint, where you can paste your YAML code and visualize the structure. +- **Integrated Development Environments (IDEs)**: Many modern IDEs, such as Visual Studio Code, Atom, and PyCharm, have built-in support for YAML. Open your YAML file in one of these IDEs to benefit from syntax highlighting and a structured view of your YAML document. +- **YAML Viewer Browser Extensions**: there are browser extensions available that can format and visualize YAML files directly in your browser. Check for extensions compatible with your preferred browser. +- **Command-Line Tools**: you can use command-line tools like `yq` or `jq` to format and view YAML content. +- **Online YAML Visualizers**: some websites offer online YAML visualizers that allow you to paste your YAML code and see a visual representation of the data structure. Search for "Online YAML Visualizer" to find such tools. + +Choose the method that best suits your preferences and workflow. + +#### Validation + +While YAML itself is not designed to be extended or derived into new languages, it is possible to create domain-specific languages (DSLs) or configuration languages based on YAML syntax. Developers can define specific rules and conventions within the YAML structure to suit the requirements of their particular domain or application. + +In essence, you can create a new language by establishing a set of guidelines for interpreting the YAML data in a specific way. This is often done in the context of configuration files or data representation for a particular software or system. Keep in mind that this is more about using YAML as a foundation and defining the semantics and rules for your specific language rather than formally deriving a new language from YAML. + +## See Also + +- [References](./../../REFERENCES.md#references) + +[CommunityJSON]: https://json-schema.org/specification#specification +[ISOJSON]: https://www.iso.org/standard/71616.html +[RFCJSON]: https://datatracker.ietf.org/doc/rfc7159 + +[XSLW3C]: https://www.w3schools.com/xml/xsl_languages.asp +[XSD]: http://msdn.microsoft.com/library/x6c1kb0s.aspx +[STLZTN]: http://msdn.microsoft.com/library/7ay27kt9.aspx + +[objects-serialization]: DataStreams/READMESerialization.md#objects-serialization +[READMECryptography]: DataStreams/READMECryptography.md#bitstream-cybersecurity +[xmlpresentation]: DataStreams/README.md#xml-based-presentation +[xml-based-validation]: DataStreams/README.md#xml-based-validation diff --git a/ExDataManagement/FunctionalProgramming/FunctionalProgramming.UnitTest/README.md b/ExDataManagement/FunctionalProgramming/FunctionalProgramming.UnitTest/README.md new file mode 100644 index 00000000..1c2eb020 --- /dev/null +++ b/ExDataManagement/FunctionalProgramming/FunctionalProgramming.UnitTest/README.md @@ -0,0 +1,15 @@ + +# Functional Programming Usage + +> Path: `ExDataManagement/FunctionalProgramming/FunctionalProgramming.UnitTest/README.md` diff --git a/ExDataManagement/FunctionalProgramming/README.md b/ExDataManagement/FunctionalProgramming/README.md new file mode 100644 index 00000000..ce02192d --- /dev/null +++ b/ExDataManagement/FunctionalProgramming/README.md @@ -0,0 +1,17 @@ + + +# Preface + +> TBD: placeholder only +> Path: ExDataManagement\FunctionalProgramming\README.md diff --git a/ExDataManagement/GraphicalData/.Media/LayeredArchitecture.png b/ExDataManagement/GraphicalData/.Media/LayeredArchitecture.png index f520b578..6830f0c6 100644 Binary files a/ExDataManagement/GraphicalData/.Media/LayeredArchitecture.png and b/ExDataManagement/GraphicalData/.Media/LayeredArchitecture.png differ diff --git a/ExDataManagement/GraphicalData/.Media/ViewModelTestContributintDependencies.png b/ExDataManagement/GraphicalData/.Media/ViewModelTestContributintDependencies.png index dfa327fb..05b789ea 100644 Binary files a/ExDataManagement/GraphicalData/.Media/ViewModelTestContributintDependencies.png and b/ExDataManagement/GraphicalData/.Media/ViewModelTestContributintDependencies.png differ diff --git a/ExDataManagement/GraphicalData/GraphicalData.Model/README.md b/ExDataManagement/GraphicalData/GraphicalData.Model/README.md new file mode 100644 index 00000000..202d2119 --- /dev/null +++ b/ExDataManagement/GraphicalData/GraphicalData.Model/README.md @@ -0,0 +1 @@ +# The Model Sub-layer of the MVVM diff --git a/ExDataManagement/GraphicalData/GraphicalData.View/README.md b/ExDataManagement/GraphicalData/GraphicalData.View/README.md new file mode 100644 index 00000000..947ea601 --- /dev/null +++ b/ExDataManagement/GraphicalData/GraphicalData.View/README.md @@ -0,0 +1 @@ +# View Implementation Examples diff --git a/ExDataManagement/GraphicalData/GraphicalData.ViewModel/README.md b/ExDataManagement/GraphicalData/GraphicalData.ViewModel/README.md new file mode 100644 index 00000000..32cde7cb --- /dev/null +++ b/ExDataManagement/GraphicalData/GraphicalData.ViewModel/README.md @@ -0,0 +1 @@ +# ViewModel Implementation Examples diff --git a/ExDataManagement/GraphicalData/ModelUnitTest/README.md b/ExDataManagement/GraphicalData/ModelUnitTest/README.md new file mode 100644 index 00000000..7e95a502 --- /dev/null +++ b/ExDataManagement/GraphicalData/ModelUnitTest/README.md @@ -0,0 +1 @@ +# Model Usage diff --git a/ExDataManagement/GraphicalData/README.md b/ExDataManagement/GraphicalData/README.md index e9d843b1..8c800924 100644 --- a/ExDataManagement/GraphicalData/README.md +++ b/ExDataManagement/GraphicalData/README.md @@ -1,5 +1,19 @@ -# Section: Graphical data - + + +# Graphical Data Preface + +> Path: `ExDataManagement/GraphicalData/README.md` > TBD: placeholder only ## Graphical Data - Part 1 diff --git a/ExDataManagement/GraphicalData/ViewModelUnitTest/README.md b/ExDataManagement/GraphicalData/ViewModelUnitTest/README.md new file mode 100644 index 00000000..eb8ad901 --- /dev/null +++ b/ExDataManagement/GraphicalData/ViewModelUnitTest/README.md @@ -0,0 +1 @@ +# ViewModel Usage diff --git a/ExDataManagement/README.md b/ExDataManagement/README.md index 65c6c024..7ebd4fc6 100644 --- a/ExDataManagement/README.md +++ b/ExDataManagement/README.md @@ -1,90 +1,72 @@ + + # External Data Management (ExDM) ## Key words -software engineering, education, learning, external data, data management, information processing, data processing, serialization, SQL, LINQ, XAML, GUI +software engineering, education, learning, external data, data management, streaming data, structural data, graphical data ## Introduction -### Subject +Computer science in general, and especially software engineering, is a field of knowledge that deals with the automation of information processing. Programs can be recognized as a driving force of that automated behavior. To achieve information processing goals programs have to implement algorithms required by applications. In other words, the programs describe how to process data, which represents information relevant to the applications in concern. Apart from the implementation of the algorithms, therefore, data management is a key issue from the point of view of automation of information processing in particular and computer science in general. -Computer science in general, and especially software engineering, is a field of knowledge that deals with the automation of information processing. Programs can be recognized as a driving force of that automated behavior. To achieve information processing goals programs have to implement algorithms required by the application. In other words, the programs describe how to process data, which represents information relevant to the application. Apart from the implementation of the algorithms, therefore, data management is a key issue from the point of view of automation of information processing in particular and computer science in general. Let's review selected language constructs, design patterns, and frameworks targeting external data management by a program. +The main aim, of the collected code snippets in this section, is to extend knowledge and improve skills related to object-oriented programming focusing on interoperability between the computing process and data visualization, archiving, and networking resources. Particular emphasis is placed on the identification of solutions that can serve as a certain design pattern with the widest possible use in the long run. -### Goal +Providing solutions valid for a long-term horizon is extremely difficult for such a dynamically developing field. Here, the experience of the author comes to the rescue. -The main aim, of the collected code snippets in this folder, is to extend knowledge and improve skills related to object-oriented programming focusing on interoperability between the computing process and data visualization, archiving, and networking resources. Particular emphasis is placed on the identification of solutions that can serve as a certain design pattern with the widest possible use in the long run. Providing solutions valid for a long-term horizon is extremely difficult for such a dynamically developing field. Here, the experience of the author comes to the rescue, who has been employing similar solutions for years using constantly changing programming tools. +To ensure a practical context of the discussion and provide sound examples, all topics are illustrated using the C\# programming language and the Visual Studio design environment. The source code used is available in the GitHub repository. Check it out from the See Also section. I believe that the proposed principles, design patterns, and scenarios are generic and may be seamlessly ported to other environments, including but not limited to different programming languages. The language and tools mentioned above have been used only to embed the discussion in a particular environment and to ensure that the examples are compliant with the programming in practice principles. -To ensure the practical context of the discussion and provide sound examples, all topics are illustrated using the C\# language and the MS Visual Studio design environment. The source code used during the course is available in this repository. I believe that the proposed principles, design patterns, and scenarios are generic and may be seamlessly ported to other environments. The language and tools mentioned above have been used only to embed the discussion in a particular environment and to ensure that the course is very practical. +## External Data -The course discusses solutions for practical scenarios regarding various aspects of external process data management. In general, the external data may be grouped as follows: +### Preface + +The external data is recognized as the data we must pull or push from outside of a boundary of the process hosting the computer program. In general, the external data may be grouped as follows: - **streaming** - bitstreams managed using content of files, or network payload - **structural** - data fetched/pushed from/to external database management systems using queries - **graphical** - data rendered on Graphical User Interface (GUI) -The external data is recognized as the data we must pull or push from outside of a boundary of the process hosting the computer program. - -## Scope - -In the `ExDataManagement` folder, the code snippets are scoped to provide sound examples covering the following topics: - -- Section 1 - Introduction - - Lesson 01 - Executive Summary - - About the course, information versus data, algorithm versus program, type - what does it mean - - Useful assets: C\# language, Visual Studio, GitHub - - Lesson 02 - Data semantics - - Type concept - - Anonymous type - - Partial types and methods - - Generics -- Section 2 - Data streams - - Lesson 03 - File and Stream Concepts - - Lesson 04 - Attributes - - Lesson 05 - Reflection - - Lesson 06 - Serialization - - Lesson 07 - Cryptography basics -- Section 3 - Structural Data](#3-section-3---structural-data) - - Lesson 08 - Anonymous function and lambda expression - - Lesson 09 - Extension method - - Lesson 10 - LINQ query and methods syntax - - Lesson 11 - LINQ to object - - Lesson 12 - LINQ to SQL -- Section 4 - Graphical data - - Lesson 13 - Graphical data (Part 1) - - [xaml](https://docs.microsoft.com/dotnet/framework/xaml-services/) - - Lesson 14 - Graphical data (Part 2) - - MVVM (Model, View, ViewModel) design pattern -- Section 5 - Summary - - Lesson 15 - Conclusion - -> **NOTE**: Unit Test role is solely code explanation rather than testing the correctness of it. - -## Lesson 15 - Conclusion - -TBD +### Data Management and Access -## See also +Data management involves the organization, storage, retrieval, communication, and manipulation of data to ensure its accuracy, security, and accessibility. It encompasses processes like data collection, storage architecture, data integration, and maintenance to support efficient analysis and decision-making. -- [Programming in Practice - Executive Summary; Udemy course; 2021][udemyPiPES]; The course explains the role of this repository as the extended examples storage that is a foundation for the Programming in Practice paradigm. The course is for all serious about the improvement of the software development skills education methodology. -- [Programming in Practice; GitBook eBook](https://mpostol.gitbook.io/pip/) - The content of this eBook is auto-generated using the markdown files collected in this repository. It is distributed online upon the open access rules. -- [Discussion panel][Discussion] -- [Postół. M, Programming Technologies 2021; Recorded lectures](https://youtube.com/playlist?list=PLC7zPvgw-YbyWXRTAe9m-ABP9YWmpLvUk) -- [Postół. M, Programming Technologies 2020; Recorded lectures](https://youtube.com/playlist?list=PLC7zPvgw-YbwOD3GaSPl6kzKhDRmmrA-9) -- [Postol. M, profile on Udemy.com][MPUdemy] +Referring to previously mentioned data kinds we need examples related to: -[Discussion]: https://github.com/mpostol/TP/discussions -[MPUdemy]: https://www.udemy.com/user/mariusz-postol/ -[udemyPiPES]: https://www.udemy.com/course/pipintroduction/?referralCode=E1B8E460A82ECB36A835 +- **streaming**: files management, bitstreams format, interoperability, cybersecurity of bitstreams, serialization +- **structural**: queries compositions, queries execution, database interoperability +- **graphical**: data rendering, data entering, events handling - +**files management**: files management functionality involves the organization, manipulation, and control of files as entities of a distributed file system. It includes tasks such as creating, opening, closing, reading, writing, deleting, and organizing files using dedicated containers, for example, directories. Key aspects of file management functionality include content protection against malicious users and metadata maintenance. + +**bitstreams format**: using bitstreams (file content) we must face a problem with how to make bitstreams human readable. The first answer is that it must be compliant and coupled with a well-known application. The application opens this bitstream as input data and exposes it to the user employing appropriate means to make the data comprehensible. Unfortunately, this approach does not apply to custom data. Therefore we should consider another approach, namely human-readable representation should be close to natural language. The first requirement for humans to understand the stream is that it has to be formatted as text. To recognize bitstream as the text an encoding must be associated directly or indirectly. The next requirement, common for both humans and computers, is that a bitstream must be associated with comprehensible syntax rules. Finally, semantics rules should be associated with the bitstream that allows to assigning of meaning to bitstreams. Shortly there have to be defined a text-based language. A domain-specific language (DSL) is a text-based language dedicated to expressing concepts and data within a specific area. Except for programming languages like Java, C#, and Python, examples of well-known and widely accepted domain-specific languages are JSON, XML, and YAML formats to name only the most crucial. + +**interoperability**: a Data Transfer Object (DTO) is a simple, lightweight data object that transfers data between software applications. DTOs are often employed to encapsulate and transport data between different system parts, such as between a client and a server to reduce network overhead. Hence, DTO is just a bitstream formatted to make applications interoperable. They typically contain only data and no business logic implementation, making them straightforward for data exchange, and to be transferred over a network as a payload of the selected protocol stack. + +**cybersecurity of bitstreams**: bitstream cybersecurity involves protecting the integrity, non-repudiation, and confidentiality of bitstreams. To ensure bitstream integrity all its users should be able to verify that the stream is not modified while it is being archived or transmitted. Non-repudiation refers to the ability to prove the origin or authorship of a bitstream. In turn, confidentiality refers to the protection of associated information from unauthorized access or disclosure. Bitstreams cybersecurity focuses on preventing unauthorized access, tampering, or reverse engineering of these bitstreams to safeguard the functionality and security of computer systems. From the programming in practice point of view, cybersecurity addresses the implementation of a hash function, digital signature, and encryption. + +**Database queries**: are commands or requests made to a database management system (DBMS) to retrieve, manipulate, or manage data stored in a database. These queries are typically written in a domain-specific language, such as SQL (Structured Query Language). The purpose of a database query is to interact with the database and perform operations like connecting, selecting specific data, updating records, inserting new data, or deleting information. A typical database query is a text that consists of statements that specify the conditions and criteria for the data retrieval or manipulation. + +**graphical user interface (GUI)**: GUI is a type of user interface that allows users to interact with electronic devices or software applications through graphical elements such as icons, buttons, windows, and menus. To handle GUI functionality allowing data rendering, data entering, and events handling is required. Data rendering refers to the process of converting raw data into a visual or presentable format for users to comprehend. Key aspects of GUI handling include converting raw data into a format suitable for further processing and adapting the presentation of data to different screen sizes or devices to ensure a consistent and effective user experience. GUIs provide a visual way for users to interact with a system, making it more intuitive and user-friendly compared to text-based interfaces. + +## Conclusion + +This section and subsections address examples of practical scenarios regarding various aspects of external data management. Referring to previously mentioned data kinds we need examples related to: + +- **streaming**: files management, bitstreams format, interoperability, cybersecurity of bitstreams, serialization +- **structural**: queries compositions, queries execution, database interoperability +- **graphical**: data rendering, data entering, events handling + +## See also + +- [References](./../REFERENCES.md#references) diff --git a/ExDataManagement/READMEConclusion.md b/ExDataManagement/READMEConclusion.md new file mode 100644 index 00000000..172cce15 --- /dev/null +++ b/ExDataManagement/READMEConclusion.md @@ -0,0 +1,11 @@ +# Conclusion + +## Preface + +## Data Streams + +## Functional Programming + +## Structural Data + +## Graphical Data diff --git a/ExDataManagement/READMEFundamentals.md b/ExDataManagement/READMEFundamentals.md new file mode 100644 index 00000000..2573e069 --- /dev/null +++ b/ExDataManagement/READMEFundamentals.md @@ -0,0 +1 @@ +# Fundamentals diff --git a/ExDataManagement/READMEPrerequisites.md b/ExDataManagement/READMEPrerequisites.md new file mode 100644 index 00000000..6059b917 --- /dev/null +++ b/ExDataManagement/READMEPrerequisites.md @@ -0,0 +1 @@ +# Prerequisites diff --git a/ExDataManagement/StructuralData/README.md b/ExDataManagement/StructuralData/README.md index 97a9f4c1..232f5c7b 100644 --- a/ExDataManagement/StructuralData/README.md +++ b/ExDataManagement/StructuralData/README.md @@ -1,4 +1,4 @@ -# Section: Structural Data +# Structural Data Preface ## Lesson 08 - Anonymous function and lambda expression diff --git a/ExDataManagement/StructuralData/StructuralData/README.md b/ExDataManagement/StructuralData/StructuralData/README.md new file mode 100644 index 00000000..5e651e0b --- /dev/null +++ b/ExDataManagement/StructuralData/StructuralData/README.md @@ -0,0 +1,4 @@ +# Structural Data - Implementation Examples + +> TBD +> Path: `ExDataManagement/StructuralData/StructuralData/README.md` diff --git a/ExDataManagement/StructuralData/StructuralDataUnitTest/README.md b/ExDataManagement/StructuralData/StructuralDataUnitTest/README.md new file mode 100644 index 00000000..3a7d5404 --- /dev/null +++ b/ExDataManagement/StructuralData/StructuralDataUnitTest/README.md @@ -0,0 +1,3 @@ +# Structural Data Usage + +> Path: `ExDataManagement/StructuralData/StructuralDataUnitTest/README.md` \ No newline at end of file diff --git a/InformationComputation/DependencyInjection/README.md b/InformationComputation/DependencyInjection/README.md index 39ee1b94..9736af62 100644 --- a/InformationComputation/DependencyInjection/README.md +++ b/InformationComputation/DependencyInjection/README.md @@ -41,7 +41,7 @@ Testing the returned data and behavior correctness is a run-time task and requir This course is not focussed on testing therefore we can introduce many simplifications making our example especially readable for examination of a selected design pattern, I mean dependency injection and a better understanding of object-oriented programming. First, we can notice that our methods don't return any data so validation in this respect is not necessary. -In the ConstructorInjection and PropertyInjection classes, we have a few methods named Alfa, Bravo, Charlie, and Delta. For the sake of simplicity let's just assume that our job is to test only the sequence in which the methods are called. For testing purposes, I have applied a tracing mechanism. One of the benefits of this approach is the possibility to reuse it also in the production environment if needed. To test the sequence in which the methods are called each one calls an instance method of an object whose reference is assigned to the private `TraceSource` field. Because it is not about testing but about programming patterns the question, which will lead our discussion is how and where to create the object that is used for tracing purposes. +In the ConstructorInjection and PropertyInjection classes, we have a few methods named Alfa, Bravo, Charlie, and Delta. For the sake of simplicity let's just assume that our job is to test only the sequence in which the methods are called. For testing purposes, I have applied a tracing mechanism. One of the benefits of this approach is the possibility to reuse it also in the production environment if needed. To test the sequence in which the methods are called each one calls an instance method of an object whose reference is assigned to the private `TraceSource` field. Because it is not about testing but about design patterns the question, which will lead our discussion is how and where to create the object that is used for tracing purposes. ## Polymorphism diff --git a/InformationComputation/LayersCommunication/README.md b/InformationComputation/LayersCommunication/README.md index b242acf5..fa80e2f5 100644 --- a/InformationComputation/LayersCommunication/README.md +++ b/InformationComputation/LayersCommunication/README.md @@ -22,7 +22,7 @@ The main challenge we will face up is how to implement all of that using layered ## Introduction To Examples -I have prepared a few examples to explain how control flow, data transfer, and event notification fulfill requirements of the bidirectional inter-layers communication. To make it as straightforward as possible, we will return to the example discussed in the dependency injection context. Let's assume that our task is to ship a library for an unknown in advance number of users. Additionally, we don't know where and how our library will be used. We only assume that it is the logic layer. The presentation and data layers have been added to make the examples more comprehensive. The examples to be investigated are located in the project `LayersCommunication`. We will use many communication patterns to validate the sequence of methods invocation. In other words, we will test the behavior of a program. The mentioned set of methods contains methods named Alfa, Bravo, Charlie, and Delta. The methods must be invoked in alphabetical order. Of course, in a production environment, we must provide only one implementation of this set but for the sake of example, we have independent implementations for proposed programming patterns. +I have prepared a few examples to explain how control flow, data transfer, and event notification fulfill requirements of the bidirectional inter-layers communication. To make it as straightforward as possible, we will return to the example discussed in the dependency injection context. Let's assume that our task is to ship a library for an unknown in advance number of users. Additionally, we don't know where and how our library will be used. We only assume that it is the logic layer. The presentation and data layers have been added to make the examples more comprehensive. The examples to be investigated are located in the project `LayersCommunication`. We will use many communication patterns to validate the sequence of methods invocation. In other words, we will test the behavior of a program. The mentioned set of methods contains methods named Alfa, Bravo, Charlie, and Delta. The methods must be invoked in alphabetical order. Of course, in a production environment, we must provide only one implementation of this set but for the sake of example, we have independent implementations for proposed design patterns. ## Program Architecture diff --git a/README.md b/README.md index 601cdc38..653553c5 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,9 @@ software engineering, sequential programming, software design, education, code e ### Preface ->The **main goal** of this repository is to turn today's students into tomorrow's advanced software developers and architects. ->The **main purpose** of this repository is to provide code extended examples for education purposes. +> **Programming in Practice** is a contribution to **Code Engineering**. Code Engineering, also known as programming, is the process of designing, developing, testing, and maintaining computer programs. It involves applying engineering principles and methodologies to writing the program text, also referred to as the code to ensure its reliability, efficiency, and scalability. Code engineering encompasses various stages, including requirements analysis, design, implementation, testing, deployment, and maintenance, all aimed at producing high-quality computer software that meets the needs of users or clients. +> The **main goal** of this repository is to turn today's students into tomorrow's advanced software developers and architects. +> The **main purpose** of this repository is to provide code extended examples for education purposes. My point is that we could distinguish between two kinds of education: @@ -53,7 +54,7 @@ The extended examples gathered in this repository focus on learning - the improv The code extended examples address the following application domains: - [**Information Computation (IC)**](InformationComputation/README.md): internal data management - all about engaging a computer (a physical device) to process information as a series of actions or steps taken to achieve a particular result or help to fulfill a task. -- [**External Data Management (ExDM)**](ExDataManagement/README.md): external (graphical, streaming, structural) data management - all about processing data managed using external resources, that is screen, DBMS (database management system), files. +- [**External Data Management (ExDM)**](ExDataManagement/README.md): external (streaming, structural, graphical) data management - all about processing data managed using external resources, that is screen, DBMS (database management system), files. - [**Adaptive Programming (AP)**](AdaptiveProgramming/README.md): language constructs, patterns, and frameworks used at the development and deployment stage to improve the future adaptability of the software. - [**Concurrent Programming (CW)**](ConcurrentProgramming/README.md): all about the programming pattern formally describing a program to execute operations as a result of nondeterministic events at run time. - [**Distributed Programming (DP)**](DistributedProgramming/README.md): all about developing inter-operable applications interconnected over the network. diff --git a/REFERENCES.md b/REFERENCES.md index e748145c..1fe32536 100644 --- a/REFERENCES.md +++ b/REFERENCES.md @@ -2,23 +2,40 @@ ## Programming in Practice +- Postol Mariusz; [Cybersecurity of External Streaming Data - Confidentiality][confidentiality] C# Corner, 2024. +- Postol Mariusz; [Cybersecurity of External Streaming Data - Integrity][CI] C# Corner, 2024. +- Postol Mariusz; [External Data Management (ExDM)][ExDM]; C# Corner, 2024 +- Postol Mariusz; [External Data - File and Stream Concepts][FileStream]; C# Corner, 2023 +- Postol Mariusz; [External Data - Attributes - Profiling Data Access][Attributes]; C# Corner, 2024 - [Programming in Practice - Information Computation; Udemy course, 2023][udemyPiPIC] - Information Computation means a process engaging a computer (a physical device) to process information as a series of actions or steps taken to achieve a particular result or help to fulfill a task. The main challenge is that information is abstract. Precisely speaking, it is a kind of knowledge that cannot be processed directly by any physical device. Generally speaking, To resolve this inconsistency two main topics are covered. The first one refers to selected aspects of information modeling using types as descendants of a coding system. The second one covers program architecture design patterns to improve the design and deployment of the computer behavior description using a program implementing an algorithm. - [Programming in Practice - Executive Summary; Udemy course; 2021][udemyPiPES]; This free course explains the role of this repository as the extended examples storage that is a foundation for the Programming in Practice paradigm. The course is for all serious about the improvement of the software development skills education methodology. - [Programming in Practice, Video Playlist of courses description on Youtube, 2023](https://www.youtube.com/playlist?list=PLC7zPvgw-Ybwya54i262_RfG5tEp2FSIt) -- [Postol. M, profile on Udemy.com][MPUdemy] +- [Postół. M, Object-Oriented Internet](https://youtube.com/playlist?list=PLC7zPvgw-YbyWss-0j_waddacgroLFTzi) This playlist on YouTube addresses research results on the systematic approach to the design of the meaningful Machine to Machine (M2M) communication targeting distributed mobile applications in the context of new emerging disciplines, i.e. Industry 4.0 and Internet of Things. - [Programming in Practice; GitBook eBook](https://mpostol.gitbook.io/pip/) - The content of this eBook is auto-generated using the Markdown files collected in this repository. It is distributed online upon the open access rules. -- [Discussion panel][Discussion] +- [GitHub repository mpostol/TP][TP] - C# in Practice - set of C# examples targeting education purpose +- [Discussion panel on mpostol/TP][Discussion] + +[CI]: https://www.c-sharpcorner.com/article/cybersecurity-of-external-streaming-data-integrity/ +[confidentiality]: https://www.c-sharpcorner.com/article/cybersecurity-of-external-streaming-data-confidentiality/ +[ExDM]: https://www.c-sharpcorner.com/blogs/external-data-management-exdm +[FileStream]: https://www.c-sharpcorner.com/article/external-data-file-and-stream-concepts +[Attributes]: https://www.c-sharpcorner.com/article/external-data-attributes-profiling-data-access/ + +## How to reach me + +- [Postol. M, profile on Udemy.com][MPUdemy] - [Postol. M, profile on GitHub.com][MPGitHub] -- [Postół. M, Programming Technologies, 2021; playlist - recorded lectures](https://youtube.com/playlist?list=PLC7zPvgw-YbyWXRTAe9m-ABP9YWmpLvUk) -- [Postół. M, Programming Technologies, 2020; playlist - recorded lectures](https://youtube.com/playlist?list=PLC7zPvgw-YbwOD3GaSPl6kzKhDRmmrA-9) -- [Postół. M, Object-Oriented Internet](https://youtube.com/playlist?list=PLC7zPvgw-YbyWss-0j_waddacgroLFTzi) This playlist on YouTube addresses research results on the systematic approach to the design of the meaningful Machine to Machine (M2M) communication targeting distributed mobile applications in the context of new emerging disciplines, i.e. Industry 4.0 and Internet of Things. +- [Postół. M, profile on LinkedIn](https://pl.linkedin.com/in/mpostol) +- [MPostol M, profile on ResearchGate](https://www.researchgate.net/profile/Mariusz-Postol) +- [MPostol M, profile on ORCID](https://orcid.org/0000-0002-9669-0565) - [Postół. M, Język C# w praktyce. Kurs video. Przetwarzanie danych zewnętrznych][vdpnt]; 2019, Helion (in polish). -- [Join me on LinkedIn](https://pl.linkedin.com/in/mpostol) [MPUdemy]:https://www.udemy.com/user/mariusz-postol/ [udemyPiPIC]: https://www.udemy.com/course/information-computation/?referralCode=9003E3EF42419C6E6B21 [udemyPiPES]: https://www.udemy.com/course/pipintroduction/?referralCode=E1B8E460A82ECB36A835 [MPGitHub]:https://github.com/mpostol + +[TP]: https://github.com/mpostol/TP [Discussion]: https://github.com/mpostol/TP/discussions [vdpnt]: https://videopoint.pl/kurs/jezyk-c-w-praktyce-kurs-video-przetwarzanie-danych-zewnetrznych-mariusz-postol,vjcprv.htm#format/w diff --git a/SUMMARY.md b/SUMMARY.md index cd11956e..204fd1b9 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,4 +1,4 @@ -# Summary +# Executive Summary * [Programming in Practice](README.md) * [References](REFERENCES.md) @@ -28,19 +28,29 @@ ## External Data Management * [Executive Summary](ExDataManagement/README.md) -* Data Semantics - - * [Generics](ExDataManagement/P02.DataSemantics/DataSemantics/Generics/Generics.md) - * [Partial types](ExDataManagement/P02.DataSemantics/DataSemantics/Partials/README.md) -* [Data Streams](ExDataManagement/P03.DataStreams/README.md) -* [Functional Programming](ExDataManagement/P04.FunctionalProgramming/FunctionalProgramming/Readme.md) -* [Structural Data](ExDataManagement/P05.StructuralData/README.md) -* [Graphical Data](ExDataManagement/P06.GraphicalData/README.md) +* Data Streams + * [Preface](ExDataManagement/DataStreams/README.md) + * [Implementation Examples](ExDataManagement/DataStreams/DataStreams/README.md) + * [Objects Serialization](ExDataManagement/DataStreams/DataStreams/READMESerialization.md) + * [Cryptography](ExDataManagement/DataStreams/DataStreams/READMECryptography.md) +* Functional Programming + * [Preface](ExDataManagement/FunctionalProgramming/README.md) + * [Implementation Examples](ExDataManagement/FunctionalProgramming/FunctionalProgramming/Readme.md) +* Structural Data + * [Preface](ExDataManagement/StructuralData/README.md) + * [Implementation Examples](ExDataManagement/StructuralData/StructuralData/README.md) + * [Usage](ExDataManagement/StructuralData/StructuralDataUnitTest/README.md) +* Graphical Data + * [Preface](ExDataManagement/GraphicalData/README.md) + * [View Implementation Examples](ExDataManagement\GraphicalData\GraphicalData.View\README.md) + * [ViewModel Implementation Examples](ExDataManagement\GraphicalData\GraphicalData.ViewModel\README.md) + * [ViewModel Usage](ExDataManagement\GraphicalData\ViewModelUnitTest\README.md) + * [Model Implementation Examples](ExDataManagement/GraphicalData/GraphicalData.Model/README.md) + * [Model Usage](ExDataManagement/GraphicalData/ModelUnitTest/README.md) ## Adaptive Programming * [Executive Summary](AdaptiveProgramming/README.md) - * Application Architecture * Composition