Visitor Pattern By Example

Visitor pattern separates some algorightm or computation work to its own class. Being able to "visit" visitable objects, different algorithm implementations can do their work without modification of original objects. Let's say a person has different kind of valuables/assets, and the value calculations could have their own implementation:

using System;
using System.Collections.Generic;

namespace ConsoleApplication1
{
    class Program
    {
        public interface IValuable  // Interface for visitable objects 
        {
            void Value(IValueCalculator calculator);
        }

        public interface IValueCalculator   // Interface for visitor implementation
        {
            decimal GetTotal();
            void Calculate(BankAccount bankAccount);
            void Calculate(Loan load);
        }

        public class NetWorthCalculator : IValueCalculator
        {
            private decimal Total = 0;

            public decimal GetTotal()
            {
                return Total;
            }

            public void Calculate(BankAccount bankAccount)
            {
                Total += bankAccount.Balance;
            }

            public void Calculate(Loan loan)
            {
                Total -= loan.Amount;
            }
        }

        public class MonthlyIncomeCalculator : IValueCalculator
        {
            private decimal Total = 0;

            public decimal GetTotal()
            {
                return Total;
            }

            public void Calculate(BankAccount bankAccount)
            {
                Total += (bankAccount.Balance * bankAccount.InterestRate) / 12;
            }

            public void Calculate(Loan loan)
            {
                Total -= loan.MonthlyPayment;
            }
        }

        public class BankAccount : IValuable
        {
            public string Name { get; set; }
            public decimal Balance { get; set; }
            public decimal InterestRate { get; set; }

            public void Value(IValueCalculator calculator)
            {
                calculator.Calculate(this);
            }
        }

        public class Loan : IValuable
        {
            public string Name { get; set; }
            public decimal Amount { get; set; }
            public decimal MonthlyPayment { get; set; }

            public void Value(IValueCalculator calculator)
            {
                calculator.Calculate(this);
            }
        }

        public class Person : IValuable // Composite pattern: Person itself is also IValuable
        {
            public string Name { get; set; }
            public List<IValuable> Valuables = new List<IValuable>();

            public void Value(IValueCalculator calculator)
            {
                foreach (IValuable item in Valuables)
                {
                    item.Value(calculator);
                }
            }
        }

        static void Main(string[] args)
        {
            Person person = new Person() { Name = "new graduate student" };
            person.Valuables.Add(new BankAccount()
            {
                Name = "Saving Account",
                Balance = 5000,
                InterestRate = 0.05m
            });
            person.Valuables.Add(new BankAccount()
            {
                Name = "Cheque Account",
                Balance = 2000,
                InterestRate = 0.01m
            });
            person.Valuables.Add(new Loan()
            {
                Name = "Student Loan",
                Amount = 5000,
                MonthlyPayment = 500
            });

            NetWorthCalculator netWorth = new NetWorthCalculator();
            person.Value(netWorth);
            Console.WriteLine("Assets' net worth is: " + netWorth.GetTotal());

            MonthlyIncomeCalculator monthlyIncome = new MonthlyIncomeCalculator();
            person.Value(monthlyIncome);
            Console.WriteLine("Monthly income from Assets: " + monthlyIncome.GetTotal().ToString("#.##"));

            Console.Read();
        }
    }
}

Run the console app will show following result:

    Assets' net worth is: 2000
    Monthly income from Assets: -477.5

The NetWorthCalculator and MonthlyIncomeCalculator are totally separated from the target IValuable objects. We can implement other type of calculation without any modification on existing code. When adding new IValuable type, the ICalculator interface and its implementations need to include the new type. Back to our example, assume the student found a job after graduation from school, paid off most of the student loan in six months and bought some Microsoft stocks. Two new IValuable types are introduced:

using System;
using System.Collections.Generic;

namespace ConsoleApplication1
{
    class Program
    {
        public interface IValuable  // Interface for visitable objects
        {
            void Value(IValueCalculator calculator);
        }

        public interface IValueCalculator   // Interface for visitor implementation
        {
            decimal GetTotal();
            void Calculate(BankAccount bankAccount);
            void Calculate(Loan load);
            void Calculate(Job stock);
            void Calculate(Stock stock);
        }

        public class NetWorthCalculator : IValueCalculator
        {
            private decimal Total = 0;

            public decimal GetTotal()
            {
                return Total;
            }

            public void Calculate(BankAccount bankAccount)
            {
                Total += bankAccount.Balance;
            }

            public void Calculate(Loan loan)
            {
                Total -= loan.Amount;
            }

            public void Calculate(Job job)
            {
                // Do nothing
            }

            public void Calculate(Stock stock)
            {
                Total += stock.Price * stock.Share;
            }
        }

        public class MonthlyIncomeCalculator : IValueCalculator
        {
            private decimal Total = 0;

            public decimal GetTotal()
            {
                return Total;
            }

            public void Calculate(BankAccount bankAccount)
            {
                Total += (bankAccount.Balance * bankAccount.InterestRate) / 12;
            }

            public void Calculate(Loan loan)
            {
                Total -= loan.MonthlyPayment;
            }

            public void Calculate(Job job)
            {
                Total += job.Salary / 12;
            }

            public void Calculate(Stock stock)
            {
                // Do nothing
            }
        }

        public class BankAccount : IValuable
        {
            public string Name { get; set; }
            public decimal Balance { get; set; }
            public decimal InterestRate { get; set; }

            public void Value(IValueCalculator calculator)
            {
                calculator.Calculate(this);
            }
        }

        public class Loan : IValuable
        {
            public string Name { get; set; }
            public decimal Amount { get; set; }
            public decimal MonthlyPayment { get; set; }

            public void Value(IValueCalculator calculator)
            {
                calculator.Calculate(this);
            }
        }

        public class Job : IValuable
        {
            public string Name { get; set; }
            public decimal Salary { get; set; }

            public void Value(IValueCalculator calculator)
            {
                calculator.Calculate(this);
            }
        }

        public class Stock : IValuable
        {
            public string Name { get; set; }
            public decimal Price { get; set; }
            public int Share { get; set; }

            public void Value(IValueCalculator calculator)
            {
                calculator.Calculate(this);
            }
        }

        public class Person : IValuable // Composite pattern: Person itself is also IValuable
        {
            public string Name { get; set; }
            public List<IValuable> Valuables = new List<IValuable>();

            public void Value(IValueCalculator calculator)
            {
                foreach (IValuable item in Valuables)
                {
                    item.Value(calculator);
                }
            }
        }

        static void Main(string[] args)
        {
            Person person = new Person() { Name = "6-month after graduation" };
            person.Valuables.Add(new BankAccount()
            {
                Name = "Saving Account",
                Balance = 20000,
                InterestRate = 0.05m
            });
            person.Valuables.Add(new BankAccount()
            {
                Name = "Cheque Account",
                Balance = 5000,
                InterestRate = 0.01m
            });
            person.Valuables.Add(new Loan()
            {
                Name = "Student Loan",
                Amount = 1000,
                MonthlyPayment = 500
            });
            person.Valuables.Add(new Job()
            {
                Name = "Software Developer",
                Salary = 60000
            });
            person.Valuables.Add(new Stock()
            {
                Name = "Microsoft",
                Price = 20,
                Share = 100
            });

            NetWorthCalculator netWorth = new NetWorthCalculator();
            person.Value(netWorth);
            Console.WriteLine("Assets' net worth is: " + netWorth.GetTotal());

            MonthlyIncomeCalculator monthlyIncome = new MonthlyIncomeCalculator();
            person.Value(monthlyIncome);
            Console.WriteLine("Monthly income from Assets: " + monthlyIncome.GetTotal().ToString("#.##"));

            Console.Read();
        }
    }
}

Run the app again and now the net worth and monthly income are both positive:

    Assets' net worth is: 26000
    Monthly income from Assets: 4587.5

The Visitor Pattern is quite useful for some scenarios. But again we use the design patterns to resolve certain problems, and let the code more extensible and maintainable. They may not be suitable for some situations and over using them may make the code unnecessarily complicated.

Invoke Generic Method Dynamically In .NET

Generic method was first introduced in .NET 2.0, and we can write method with dynamic types like:

using System;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;

class Test
{
    public class Foo
    {
        public string Name { get; set; }
        public Foo(string name)
        {
            Name = "Foo" + name;
        }
    }
    public static List<T> ParseValues<T>(string[] data)
    {
        List<T> values = new List<T>();
        Type type = typeof(T);
        if (type == typeof(int))
        {
            foreach (string str in data)
                values.Add((T)((object)int.Parse(str)));
        }
        else if (type == typeof(double))
        {
            foreach (string str in data)
                values.Add((T)((object)double.Parse(str)));
        }
        else if (type == typeof(Foo))
        {
            foreach (string str in data)
                values.Add((T)((object)new Foo(str)));
        }
        return values;
    }

    static void Main()
    {
        string[] values = new string[] { "1", "2", "3" };
        List<int> numbers = ParseValues<int>(values);
        List<double> doubles = ParseValues<double>(values);
        List<Foo> foos = ParseValues<Foo>(values);

        //Print 1 2 3 1 2 3 Foo1 Foo2 Foo3
        numbers.ForEach(i => Console.Write(i + " "));
        doubles.ForEach(i => Console.Write(i + " "));
        foos.ForEach(i => Console.Write(i.Name + " "));

        Console.Read();
    }
}

In order to invoke a generic method you have to declare the type statically, and it will be initiated at compile time. Is there a way to get a generic type dynamically? We can do something like:

    Type genericListType = typeof(List<>).MakeGenericType(typeof(double));
    var list = Activator.CreateInstance(genericListType);

But that doesn't resolve our problem because we can't determine the T type at compile time to invoke the method of public static List<T> ParseValues<T>(string[] data).

One way I can think of is use the reflection and invoke the method dynamically:

using System;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;

class Test
{
    public class Foo
    {
        public string Name { get; set; }
        public Foo(string name)
        {
            Name = "Foo" + name;
        }
    }
    public static List<T> ParseValues<T>(string[] data)
    {
        List<T> values = new List<T>();
        Type type = typeof(T);
        if (type == typeof(int))
        {
            foreach (string str in data)
                values.Add((T)((object)int.Parse(str)));
        }
        else if (type == typeof(double))
        {
            foreach (string str in data)
                values.Add((T)((object)double.Parse(str)));
        }
        else if (type == typeof(Foo))
        {
            foreach (string str in data)
                values.Add((T)((object)new Foo(str)));
        }
        return values;
    }

    static void Main()
    {
        string[] values = new string[] { "1", "2", "3" };
        List<int> numbers = ParseValues<int>(values);
        List<double> doubles = ParseValues<double>(values);
        List<Foo> foos = ParseValues<Foo>(values);

        //Print 1 2 3 1 2 3 Foo1 Foo2 Foo3
        numbers.ForEach(i => Console.Write(i + " "));
        doubles.ForEach(i => Console.Write(i + " "));
        foos.ForEach(i => Console.Write(i.Name + " "));
        Console.WriteLine();

        Type numberType = typeof(int);
        MethodInfo genericMethod = typeof(Test).GetMethod("ParseValues").MakeGenericMethod(numberType);
        var dynamicList = genericMethod.Invoke(null, new object[] { values });

        // Print 1 2 3
        foreach (var item in dynamicList as IEnumerable)
            Console.Write(item + " ");

        Type fooType = typeof(Foo);
        genericMethod = typeof(Test).GetMethod("ParseValues").MakeGenericMethod(fooType);
        dynamicList = genericMethod.Invoke(null, new object[] { values });

        // Print Foo1 Foo2 Foo3
        foreach (var item in dynamicList as IEnumerable)
            Console.Write((item as Foo).Name + " "); 

        Console.Read();
    }
}

Bear in mind that using Reflection is costly and it would affect performance significantly.

ASP.NET Substitute Custom Control In Output-cached Page

ASP.NET output cache is great. For dynamic content inside output-cached pages, ASP.NET 2.0 introduces Substitution concept, where the dynamic content is retrieved and substituted for the Substitution control at run time.

Usually Substitution content is just a string. But we can extend it to render a custom server control. For example there's a custom control named MyCustomControl that needs to be displayed dynamically in a output-cached page. First we need to declare a Substitution:

<asp:Substitution runat="server" ID="subControl" MethodName="GetControlString" />

Inside the GetControlString static method:

    public static string GetControlString(HttpContext context)
    {
        StringWriter output = new StringWriter();
        Page pageHolder = new Page();
        MyCustomControl control = MyCustomControl();
        //control.SetParameters();
        //pageHolder.Response.ContentEncoding = System.Text.Encoding.GetEncoding("UTF-8");
        pageHolder.Controls.Add(control);

        context.Server.Execute(pageHolder, output, false);
        context.Response.ContentEncoding = Encoding.GetEncoding("UTF-8");

        return output.ToString();
    }

[2009/06 Updated] HttpContext.Response.ContentEncoding needs to be defined specifically if you encounter the wrong character encoding issue inside the custom control.

ASP.NET Cache Info Page

This is an ASP.NET page to show Cache usage of current w3wp process:



CacheInfo.aspx:

<%@ Page Language="C#" AutoEventWireup="true" CodeBehind="CacheInfo.aspx.cs" Inherits="CacheInfo" %>

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head runat="server">
    <title>Cache Info/Test Page</title>
</head>
<body>
    <form id="form1" runat="server">
    <div>
        <div>
            <p style="text-align: center">
                <asp:Label ID="Label4" runat="server" Text="Cache Info" Font-Size="Large" Font-Bold="true"></asp:Label></p>
            <p>
                <asp:Label ID="lblSysInfo" runat="server" EnableViewState="false"></asp:Label>
                <asp:Label ID="lblInfo" runat="server" EnableViewState="false"></asp:Label>
            </p>
            <p>
                <asp:Label ID="lblCache" runat="server" EnableViewState="false"></asp:Label><br />
                <asp:GridView ID="gvCache" runat="server" AutoGenerateColumns="False" BorderWidth="1px"
                    BackColor="White" CellPadding="4" BorderStyle="Solid" BorderColor="#3366CC" Font-Size="Small"
                    AlternatingRowStyle-ForeColor="ActiveCaption" EnableViewState="false">
                    <HeaderStyle ForeColor="White" BackColor="#003399" HorizontalAlign="Left"></HeaderStyle>
                    <Columns>
                        <asp:TemplateField ItemStyle-Width="20" ItemStyle-Font-Size="X-Small">
                            <ItemTemplate>
                                <%# Container.DataItemIndex + 1 %>
                            </ItemTemplate>
                        </asp:TemplateField>
                        <asp:BoundField HeaderText="Key" DataField="Key" ReadOnly="true" ItemStyle-Width="420px"></asp:BoundField>
                        <asp:BoundField HeaderText="Type" DataField="Type" ReadOnly="true" ItemStyle-Width="180px"></asp:BoundField>
                        <asp:BoundField HeaderText="Note" DataField="Note" ReadOnly="true" ItemStyle-Width="180px"></asp:BoundField>
                    </Columns>
                    <EmptyDataTemplate>
                        <asp:Label ID="lblEmptyMessage" runat="server" Text="No feature found."></asp:Label>
                    </EmptyDataTemplate>
                </asp:GridView>
            </p>
            <p>
                <asp:Label ID="Label3" runat="server" Text="Add cache with size in MB"></asp:Label>
                <asp:DropDownList ID="ddlCacheSize" runat="server" AutoPostBack="false">
                    <asp:ListItem>50</asp:ListItem>
                    <asp:ListItem>100</asp:ListItem>
                    <asp:ListItem>200</asp:ListItem>
                    <asp:ListItem>300</asp:ListItem>
                    <asp:ListItem>400</asp:ListItem>
                    <asp:ListItem>500</asp:ListItem>
                    <asp:ListItem>600</asp:ListItem>
                    <asp:ListItem>700</asp:ListItem>
                    <asp:ListItem>800</asp:ListItem>
                    <asp:ListItem>900</asp:ListItem>
                    <asp:ListItem>1000</asp:ListItem>
                    <asp:ListItem>2000</asp:ListItem>
                    <asp:ListItem>3000</asp:ListItem>
                    <asp:ListItem>4000</asp:ListItem>
                    <asp:ListItem>5000</asp:ListItem>
                    <asp:ListItem>6000</asp:ListItem>
                    <asp:ListItem>7000</asp:ListItem>
                    <asp:ListItem>8000</asp:ListItem>
                    <asp:ListItem>9000</asp:ListItem>
                    <asp:ListItem>10000</asp:ListItem>
                </asp:DropDownList>&nbsp;&nbsp;
                <asp:Label ID="Label1" runat="server" Text="Cache Expire in seconds"></asp:Label>
                <asp:DropDownList ID="ddlExpir" runat="server" AutoPostBack="false">
                    <asp:ListItem>10</asp:ListItem>
                    <asp:ListItem>20</asp:ListItem>
                    <asp:ListItem>30</asp:ListItem>
                    <asp:ListItem>40</asp:ListItem>
                    <asp:ListItem>50</asp:ListItem>
                    <asp:ListItem>60</asp:ListItem>
                    <asp:ListItem>70</asp:ListItem>
                    <asp:ListItem>80</asp:ListItem>
                    <asp:ListItem>90</asp:ListItem>
                    <asp:ListItem>100</asp:ListItem>
                    <asp:ListItem>110</asp:ListItem>
                    <asp:ListItem>120</asp:ListItem>
                    <asp:ListItem>180</asp:ListItem>
                </asp:DropDownList>&nbsp;&nbsp;
                <asp:Button ID="btnAddCache" runat="server" Text="Add Test Cache" OnClick="btnAddCache_click" />
            </p>
        </div>
    </div>
    </form>
</body>
</html>

CacheInfo.aspx.cs:

using System;
using System.Collections;
using System.Collections.Generic;
using System.Web;
using System.Web.UI.WebControls;
using System.Web.Caching;
using System.Text;
using System.Data;

public partial class CacheInfo : System.Web.UI.Page
{
    protected class CacheInfoItem
    {
        public string Key { get; set; }
        public string Type { get; set; }
        public string Note { get; set; }
        public CacheInfoItem(string key, string type)
        {
            Key = key;
            Type = type;
        }
    }

    protected class CacheData
    {
        public Byte[] Data { get; set; }
        public CacheData(int totalBytes)
        {
            Data = new Byte[totalBytes];
            for (int i = 0; i < totalBytes; i += 123)
            {
                Data[i] = (byte)(i % 256);
            }
        }
    }

    protected string TestCacheKey = "Cache_Test_Key";

    protected void btnAddCache_click(object sender, EventArgs e)
    {
        try
        {
            int totalBytes = int.Parse(ddlCacheSize.SelectedValue) * 1024 * 1024;
            CacheData data = new CacheData(totalBytes);
            int seconds = int.Parse(ddlExpir.SelectedValue);
            HttpRuntime.Cache.Insert(TestCacheKey, data, null, DateTime.Now.AddSeconds(seconds), Cache.NoSlidingExpiration);
            lblInfo.Text = "Successfully added test data to cache (expire in " + seconds + " seconds).";
        }
        catch (Exception ex)
        {
            lblInfo.Text = "Insert Cache error: " + ex.Message;
        }
    }

    protected override void OnPreRender(EventArgs e)
    {
        base.OnPreRender(e);
        List<CacheInfoItem> cacheItems = new List<CacheInfoItem>();

        // Populate System Info
        System.Diagnostics.Process currentProcess = System.Diagnostics.Process.GetCurrentProcess();
        long totalMemBytes = currentProcess.WorkingSet64;
        DateTime startTime = currentProcess.StartTime;
        Cache cache = HttpRuntime.Cache;
        StringBuilder sbInfo = new StringBuilder();
        sbInfo.Append(Server.MachineName + " " + currentProcess.MachineName + "  ");

        sbInfo.Append(currentProcess.ProcessName + " start time: " + startTime.ToString("yyy-MM-dd HH:mm") + "  ");
        sbInfo.Append("Process memory usage (MB): " + totalMemBytes / (1024 * 1024) + "<br>");
        lblSysInfo.Text = sbInfo.ToString();

        // Populate Cache Info
        sbInfo = new StringBuilder();
        if (cache != null)
        {
            foreach (DictionaryEntry item in cache)
            {
                try
                {
                    Type type = item.Value.GetType();
                    string typeInfo = type.ToString();

                    CacheInfoItem ci = new CacheInfoItem(item.Key.ToString(), typeInfo);
                    cacheItems.Add(ci);

                    if (type == Type.GetType("System.DateTime"))
                    {
                        ci.Note = "Time: " + Convert.ToDateTime(HttpRuntime.Cache[item.Key.ToString()]).ToString("yyy-MM-dd HH:mm");
                    }
                    else if (type == Type.GetType("System.String"))
                    {
                        ci.Note = "Value: " + cache[item.Key.ToString()].ToString();
                    }
                    else
                    {
                        Object obj = cache[item.Key.ToString()];
                        if (obj != null)
                        {
                            if (obj is IList)
                            {
                                ci.Note = "Item Count: " + ((IList)obj).Count.ToString();
                            }
                            else if (obj is DataSet && (DataSet)obj != null && ((DataSet)obj).Tables[0] != null)
                            {
                                ci.Note = "Item Count: " + ((DataSet)obj).Tables[0].Rows.Count.ToString();
                            }
                            else if (obj is DataTable && (DataTable)obj != null)
                            { 
                                ci.Note = "Item Count: " + ((DataTable)obj).Rows.Count.ToString(); 
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    sbInfo.Append("Get cache info error for " + item.Key + ":" + ex.Message + "<br>");
                    throw;
                }
            }
        }
        lblInfo.Text += sbInfo.ToString();

        lblCache.Text = "Total Cache item count: " + cacheItems.Count.ToString();
        cacheItems.Sort((i, j) => i.Key.CompareTo(j.Key));
        gvCache.DataSource = cacheItems;
        gvCache.DataBind();
    }
}

LINQ SelectMany Usage

Select and SelectMany are two projection operators in LINQ. Select simply goes through a sequence of elements (IEnumerable), and for each input element it exactly projects one output element. On the contrast, SelectMany can projects 0, 1 or multiple elements for each input elements. It works something like this, for each input element, I can do something based on the element context or just fixed logic, then produces some new elements to return.

For instance, each instructor offers many courses in the school, Instructors.SelectMany(i => i.Courses) means for each instructor in the Instructors list, return his/her Courses, so you get all courses for all instructors.

A SelectMany overloading method accepts the second parameter which gives us more power to control the output. Let’s look at a statement:

Instructors.SelectMany(i => i.Courses, (i, c) => new { InstructorName = i.Name, CourseName = c } ).

The first parameter "i => i.Courses" will get all Courses for each instructor, and transform it to the second parameter, so inside "(i, c)", "i" is the original input element just like what we have in "i => i.Courses", and "c" here is each element sent from "i.Courses". What the statement does is go through each instructor’s courses, and return the instructor’s name and course name with anonymous type of object.

Following code example demos a few usages of SelectMany, including how you can join a sequence to another sequence, and do you your customized logic inside the lambda function:

using System;
using System.Linq;
using System.Collections.Generic;

class Program
{
    class Instructor
    {
        public string Name { get; set; }
        public string DeptID { get; set; }
        public string[] Courses { get; set; }
    }

    class Department
    {
        public string ID { get; set; }
        public string Name { get; set; }
    }

    static void Main()
    {
        var departments = new List<Department>()
            {
                new Department() { ID = "EE", Name = "Electrical Engineering" },
                new Department() { ID = "CS", Name = "Computer Science" },
                new Department() { ID = "ME", Name = "Mechanical Engineering" }
            };
        var instructors = new List<Instructor>()
            {
                new Instructor() { Name = "EE1", DeptID = "EE", Courses = new string [] { "EE101", "EE102", "EE103" } },
                new Instructor() { Name = "EE2", DeptID = "EE", Courses = new string [] { "EE201", "EE202", "EE203" } },
                new Instructor() { Name = "CS1", DeptID = "CS", Courses = new string [] { "CS101", "CS102", "CS103" } },
                new Instructor() { Name = "CS2", DeptID = "CS", Courses = new string [] { "CS201", "CS202", "CS203" } },
                new Instructor() { Name = "ME1", DeptID = "ME", Courses = new string [] { "ME101", "ME102", "ME103" } },
                new Instructor() { Name = "ME2", DeptID = "ME", Courses = new string [] { "ME201", "ME202", "ME203" } },
            };
        var CSInstructors = instructors.Where(i => i.DeptID == "CS").Select(i => "CS Instructor: " + i.Name).ToArray();
        // CSInstructors: { "CS Instructor: CS1", "CS Instructor: CS2" }
  
        var CSCourses = instructors.Where(i => i.DeptID == "CS").SelectMany(i => i.Courses).ToArray();
        // CSCourses.Count = 6
        // CSCourses: { "CS101", "CS102", "CS103", "CS201", "CS202", "CS203" }

        var CSInstructorCourses = instructors.Where(i => i.DeptID == "CS").
            SelectMany(i => i.Courses, (i, c) => i.Name + ":" + c).ToArray();
        // CSInstructorCourses.Count = 6
        // CSInstructorCourses: { "CS1:CS101", "CS1:CS102", "CS1:CS103", "CS2:CS201", "CS2:CS202", "CS2:CS203" } 

        var instructorDepartment = instructors.SelectMany(i => departments.Where(d => d.ID == i.DeptID),
            (i, d) => i.Name + ":" + d.Name).ToArray(); 
        // instructorDepartment.Count = 6
        // instructorDepartment: { "EE1:Electrical Engineering", "EE2:Electrical Engineering", "CS1:Computer Science" ... }

        var departmentCourses = instructors.SelectMany(
            (i, index) => i.Courses.Select(c => new { Index = index, CourseName = c}),
            (i, c) => new { Course = c.CourseName, Instructor = i.Name, 
                DeptName= departments.Find(d => d.ID == i.DeptID).Name, Index = c.Index, }).ToArray();
        /* departmentCourses.Count = 18
         * departmentCourses: 
            { Course = "EE101", Instructor = "EE1", DeptName = "Electrical Engineering", Index = 0 }
            { Course = "EE102", Instructor = "EE1", DeptName = "Electrical Engineering", Index = 0 }
            { Course = "EE103", Instructor = "EE1", DeptName = "Electrical Engineering", Index = 0 }
            { Course = "EE201", Instructor = "EE2", DeptName = "Electrical Engineering", Index = 1 }
            { Course = "EE201", Instructor = "EE2", DeptName = "Electrical Engineering", Index = 1 }
            ...
            { Course = "ME202", Instructor = "ME2", DeptName = "Mechanical Engineering", Index = 5 }
            { Course = "ME203", Instructor = "ME2", DeptName = "Mechanical Engineering", Index = 5 }
         */

        var courses101 = instructors.SelectMany(
            i => {
                List<string> filterCourses = new List<string>();
                foreach(var course in i.Courses)
                {
                    if (course.EndsWith("101"))
                        filterCourses.Add(course);
                }
                return filterCourses;
            }, 
            (i, c) => {
                var dept = departments.Find(d => d.ID == i.DeptID);
                var deptName = (dept == null) ? "N/A" : dept.Name;
                return new { Course = c, Instructor = i.Name, DeptName = deptName };
            }).ToArray();
        /* courses101.Count = 3
         * courses101:
            { Course = "EE101", Instructor = "EE1", DeptName = "Electrical Engineering" }
            { Course = "CS101", Instructor = "CS1", DeptName = "Computer Science" }
            { Course = "ME101", Instructor = "ME1", DeptName = "Mechanical Engineering" }
         */

        Console.Read();
    }
}

The Cost Of Linq And IQueryable

The Language Integrated Query (Linq) is part of the .NET 3.5. We can easily do Linq queries against IEnumerable objects using very concise syntax by passing a delegate or the new Lambda expression. You can even create your own Linq Provider and implement the IQueryable interface, building something like Linq to Google or Linq to MySql.

It's super great, right? Basically yes, but remember a lot of new features from Microsoft have a cost. In my previous post we discussed the Linq deferred execution issue. Today let's do some tests to see how expensive the Linq queries are. Below is the simple test code. We build a simple key/Value collection (array). Because all collections including array have implemented IEnumerable interface, we can query the objects inside the array with Linq. Furthermore, we can convert the array object to be IQueryable and do query over IQueryable.

using System;
using System.Linq;
using System.Collections.Generic;
using System.Diagnostics;

class Program
{
    // A simple class presenting key/value pair
    class KeyValue
    {
        public KeyValue(string key, int value)
        {
            Key = key;
            Value = value;
        }
        public string Key { get; set; }
        public int Value { get; set; }
    }

    // Simple linear search 
    static KeyValue LinearSearch(KeyValue[] keyValues, string key)
    {
        foreach (KeyValue item in keyValues)
        {
            if (item.Key == key)
                return item;
        }
        return null;
    }

    /// <summary>
    /// Compare search performance with three methods:
    /// 1. Manually go through each object inside collection
    /// 2. Linq search by IEmuerable<T>.Where(lambda_expression);
    /// 3. Convert to IQueryable and then IQueryable<T>.Where(lambda_expression)
    /// </summary>
    /// <param name="keyValues">The array to be search on</param>
    /// <param name="key">Search object by KeyValue.Key</param>
    /// <param name="repeat">Number for repeating the search</param>
    static void SearchTest(KeyValue[] keyValues, string key, int repeat)
    {
        Stopwatch watch = new Stopwatch();

        // Test linear search
        watch.Start();
        for (int i = 0; i < repeat; i++)
            LinearSearch(keyValues, key);
        watch.Stop();
        Console.WriteLine("Linear search over {0} objects (repeating {1} times) takes {2} ms",
            keyValues.Count(), repeat, watch.ElapsedMilliseconds);
        watch.Reset(); watch.Start();

        // Test Linq search
        for (int i = 0; i < repeat; i++)
            keyValues.FirstOrDefault(item => item.Key == key);
        watch.Stop();
        Console.WriteLine("Linq search over {0} objects (repeating {1} times) takes {2} ms",
            keyValues.Count(), repeat, watch.ElapsedMilliseconds);

        // Test IQueryable search
        IQueryable<KeyValue> queryables = keyValues.AsQueryable();
        watch.Reset(); watch.Start();
        for (int i = 0; i < repeat; i++)
            queryables.FirstOrDefault(item => item.Key == key);
        watch.Stop();
        Console.WriteLine("Linq IQueryable search over {0} objects (repeating {1} times) takes {2} ms",
            keyValues.Count(), repeat, watch.ElapsedMilliseconds);
        Console.WriteLine();
    }


    static void Main()
    {
        // Build test data
        KeyValue[] keyValues1 =
            Enumerable.Range(0, 1000).Select(i => new KeyValue(i.ToString(), i)).ToArray();
        KeyValue[] keyValues2 =
            Enumerable.Range(0, 2000).Select(i => new KeyValue(i.ToString(), i)).ToArray();
        KeyValue[] keyValues3 =
            Enumerable.Range(0, 3000).Select(i => new KeyValue(i.ToString(), i)).ToArray();

        // Start Test
        SearchTest(keyValues1, "5000", 1000);
        SearchTest(keyValues2, "10000", 1000);
        SearchTest(keyValues3, "20000", 1000);

        Console.Read();
    }
}

Result:


As expected Linq to objects is slightly slower than the regular linear search, and we can assume Linq uses linear search internally. But IQueryable search is much slower than others. Why that happened? Not like Linq to object where Lambda expression is passing as a function delegate, Lambda expression passed to the IQueryable is converted to Expression tree which is a quite expensive.

The interesting thing is that explicit delegate passed to IQueryable won't be converted to Expression tree. In following code there's no noticeable performance difference between the two queries.

    KeyValue keyValueObj = keyValues.FirstOrDefault(
        delegate(KeyValue obj) { return obj.Key == key; });
    IQueryable<KeyValue> queryables = keyValues.AsQueryable();
    keyValueObj = queryables.FirstOrDefault(
        delegate(KeyValue obj) { return obj.Key == key; });

Conclusion: Linq to objects are working fine for .NET collections, but be cautious to make a collection as IQueryable (collection.AsQueryable() method) if you are using Lambda expression inside the IQueryable queries, because there's performance degradation for it.

.NET Linq Deferred Execution

The new Linq (Language Integrated Query) feature shipped with .NET 3.5 is fantastic. With Linq now we can write more concise and meaningful queries. Like anonymous method pitfall we discussed in previous post, Linq has a similar behavior called deferred execution; it could be problematic if you are not aware of that. Let's look at following code:

using System;
using System.Linq;
using System.Collections.Generic;

class Program
{
    static void Main()
    {
        // Test Data
        string[] names = new string [] { "NameA", "NameB"};
        string _name = "NameA";

        // Query by delegate
        IEnumerable<string> searchName1 = names.Where(
            delegate(string name)
            {
                return name == _name;
            });
        // Query by Lambda expression
        IEnumerable<string> searchName2 = names.Where(name => name == _name);

        // Rename the search keyword
        _name = "NameB";

        // Redo the queries
        IEnumerable<string> searchName3 = names.Where(
            delegate(string name)
            {
                return name == _name;
            });
        IEnumerable<string> searchName4 = names.Where(name => name == _name);

        Console.WriteLine("{0} \t {1} \t {2} \t {3}",
            searchName1.First(), searchName2.First(), searchName3.First(), searchName4.First());

        Console.Read();
    }
}

What's the result? you may think it must be "NameA NameA NameB NameB". But you will get "NameB NameB NameB NameB" instead if you run the console application.

Why's that? Because Linq's Where search is by default a deferred execution function. For example, the statement

string searchName2 = names.Where(name => name == _name);

is telling the compiler that we have a Lambda expression attached to the Where search. But it's not invoked until we are actually reading data from the search result. So searchName1, searchName2, searchName3 and searchName4 in our case are the same because they all compare the same value when their condition is examined.

How to avoid this issue? Reading data immediately after the query such as looping through the data inside the IEnumerable collection. The other way is use ToList() or ToArray() methods to force immediate execution of a Linq query.

Following Linq methods have deferred execution behavior:

Except, Take, TakeWhile, Skip, SkipWhile, Where

While the others don't have such behavior, and will be executed immediately:

Any, Average, Contains, Count, First, FirstOrDefault, Last, LastOrDefault, Single,
 SingleOrDefault, Sum, Max, Min, ToList, ToArray, ToDictionary, ToLookup

How to remember all this? The tip is to look at the return type of the method. It would be deferred execution if the return type is IEnumerable<TSource>. Why? Yield return is used by those methods. That's the root cause of the deferred execution.

C# Anonymous Delegate Pitfall

What is output of following code?

using System;
using System.Collections.Generic;
using System.Text;

namespace AnonymousDelegate
{
class Program
{
static void Main(string[] args)
{
   Console.WriteLine("AnonymousDelegate1:");
   AnonymousDelegate1();
   Console.WriteLine();

   Console.WriteLine("AnonymousDelegate2:");
   AnonymousDelegate2();
   Console.WriteLine();

   Console.WriteLine("AnonymousDelegate3:");
   AnonymousDelegate3();
   Console.Read();
}

static void AnonymousDelegate1()
{
   List<Action> actions = new List<Action>();
   for (int i = 0; i < 3; i++)
   {
       actions.Add(delegate() { Console.WriteLine(i); });
   }
   foreach (Action action in actions)
   {
       action();
   }
}

static void AnonymousDelegate2()
{
   List<Action> actions = new List<Action>();
   for (int i = 0; i < 3; i++)
   {
       int temp = i;
       actions.Add(delegate() { Console.WriteLine(temp); });
   }
   foreach (Action action in actions)
   {
       action();
   }
}

static void AnonymousDelegate3()
{
   List<Action> actions = new List<Action>();
   int temp;
   for (int i = 0; i < 3; i++)
   {
       temp = i;
       actions.Add(delegate() { Console.WriteLine(temp); });
   }
   foreach (Action action in actions)
   {
       action();
   }
}
}
}

The result may surprise you:

AnonymousDelegate1:
3
3
3

AnonymousDelegate2:
0
1
2

AnonymousDelegate3:
2
2
2

Why is inconsistent for each test? That stems from the way how .NET handle anonymous delegate or anonymous method.

.NET compiler will create a delegate and a static method for an anonymous delegate, just like a traditional delegate. There's no anonymous concept in intermediate language (IL) in any .NET assembly.

The thing becomes more interesting when a variable used inside anonymous delegate/method is declared outside the anonymous delegate/method, which is called closure scenario. .NET compiler will wrap the anonymous method and the variable to a sealed class: anonymous method becomes a class method, variable becomes a public member. We can examine this by looking inside the IL code (click to see bigger picture):



As we can see there're 3 inner classes (<>c__DisplayClass2/5/9 in red boxes) generated for 3 anonymous methods. All three classes have the same structure:

private sealed class ComiplerGeneratedClass
{
    public int i;
    public void ActionMethod()
  {
     Console.WriteLine(this.i);
  }
}

That makes sense but why having different results? Let's check the IL code for AnonymousDelegate1:



We can see that only one AnonymousClass object is created, and AnonymousClass.i is used for looping condition (for loop) check.

But when we look at the IL code for method of AnonymousDelegate2, we found AnonymousClass is created 3 times inside the for loop. To be more readable, we translate the IL code back to C# code for all three test methods:

      private sealed class AnonymousClass // Generated by compiler
     {
         public int i;
         public void ActionMethod()
         {
             Console.WriteLine(this.i);
         }
     }

     static void AnonymousDelegate1()
     {
         AnonymousClass anonymousClass = new AnonymousClass();
         List<Action> actions = new List<Action>();
         anonymousClass.i = 0;
         for (; anonymousClass.i < 3; anonymousClass.i++)
         {
             Action action = new Action(anonymousClass.ActionMethod);
             actions.Add(action);
         }
         foreach (Action action in actions)
         {
             action.Invoke();
         }
     }

     static void AnonymousDelegate2()
     {
         List<Action> actions = new List<Action>();
         for (int i = 0; i < 3; i++)
         {
             AnonymousClass anonymousClass = new AnonymousClass();
             anonymousClass.i = i;
             Action action = new Action(anonymousClass.ActionMethod);
             actions.Add(action);
         }
         foreach (Action action in actions)
         {
             action.Invoke();
         }
     }

     static void AnonymousDelegate3()
     {
         AnonymousClass anonymousClass = new AnonymousClass();
         List<Action> actions = new List<Action>();
         for (int i = 0; i < 3; i++)
         {
             anonymousClass.i = i;
             Action action = new Action(anonymousClass.ActionMethod);
             actions.Add(action);
         }
         foreach (Action action in actions)
         {
             action.Invoke();
         }
     }

Now the answer is clear. .NET compiler generates different code depending on the scale of closure variable. It looks very confusing at the beginning and it's easy to write buggy code if we don't understand this.

We have to put it in mind that a real class is created, and a public member is used to store a closure variable for anonymous methods with closure. If not sure how the closure variable is handled, use minimum scale of local variable and pass it to the anonymous method, like what AnonymousDelegate2 does; that ensures each anonymous method using independent variable, but be aware of consuming more resource in such case because more anonymous classes are generated at run time.

Remove Files Read-only Attribute In Windows

A Unix machine that's not in our control is running a daily process of copying files in folders to a Windows server under our control by rsync. But some of the files have read-only permission in the target Windows machine, and these read-only files in Windows cause some problem in a separate file-handling process.

To remove a file read-only permission in Windows we can use attrib command:

attrib -r filename

About DOS command will remove all attributes from the file. But attrib can only handle single file. In Unix/Linux we can simply call "chomd -R 777 folder" to recursively change the permission of all files in the folder, but not that easy in Windows.

To do this unix-simple-task in Windows, I created a console application:

using System;
using System.Collections.Generic;
using System.Text;
using System.Configuration;
using System.IO;
using System.Security.AccessControl;

namespace Bell.Sympatico.Utilities.ReadOnlyRemover
{
   /// <summary>
   /// This console application is to remove file's read only attribute
   /// The user who runs the application needs the modify permission for the files
   /// Application configuration file name: ReadOnlyRemover.exe.config
   ///
   /// Usage:
   /// 1.  Process all files in a folder and its subfolders.
   ///     Specify the folder (RootFolder entry) in App.config configuration file
   ///     C:\ApplicationFolder>ReadOnlyRemover.exe /all
   /// 2.  Process feed and/or log files.
   ///     Specify the file name for Feed files (FeedFile entry in configuration file) and/or
   ///     Specify the file name for Log files (LogFile entry in configuration file)
   ///     C:\ApplicationFolder>ReadOnlyRemover.exe
   /// 3.  Process all files in a folder and its subfolders. But the folder name is
   ///     provided in the arguments. The folder name can be a share folder in network.
   ///     You need the assign read/write permission for the network shared folder in
   ///     such case.
   ///     C:\ApplicationFolder>ReadOnlyRemover.exe /all c:\test
   ///     C:\ApplicationFolder>ReadOnlyRemover.exe /all \\NetworkShared\Feed1
   ///
   /// A application log file will be created or appended during the process.
   /// You can define the application log file name in configuration file (AppLogFile entry)
   /// </summary>
   class ReadOnlyRemover
   {
       static StreamWriter sw = null; // Write the logging in a file.
       static bool doLog = true; // Do we need to do the logging during the processing?

       /// <summary>
       /// The entry point of the application
       /// </summary>
       /// <param name="args">Arguments</param>
       static void Main(string[] args)
       {
           bool scanAll = false; // A flag if check all files inside a folder
           if (args.Length > 0 && (args[0].ToLower() == "/all" || args[0].ToLower() == "all"))
           {
               scanAll = true;
           }

           string rootFolder = ConfigurationManager.AppSettings["RootFolder"];
           if (string.IsNullOrEmpty(rootFolder) || !Directory.Exists(rootFolder))
           {
               rootFolder = "C:/Feed/";    // Default folder name
           }
           else if (!rootFolder.EndsWith("/") && !rootFolder.EndsWith("\\"))
           {
               rootFolder = rootFolder + "/";
           }

           if (scanAll && args.Length > 1)
           {
               string argument = args[1];
               if (Directory.Exists(argument))
               {
                   rootFolder = argument;
               }
               else
               {
                   Console.WriteLine("Specified folder invalid!");
                   return;
               }
           }

           string appLogFile = ConfigurationManager.AppSettings["AppLogFile"];
           if (string.IsNullOrEmpty(appLogFile))
           {
               appLogFile = "ReadOnlyRemover.log.txt"; // Default application log name
           }

           try
           {
               sw = new StreamWriter(appLogFile, true);
               sw.WriteLine("Process starts at " + DateTime.Now.ToString("yyyy/MM/dd HH:mm"));
               sw.Flush();
           }
           catch (Exception ex)
           {
               doLog = false;  // Do not write the log in the file in such case
               Console.WriteLine("Could not create or open application log file. Error: " + ex.Message);
               Console.WriteLine("Continue to process without logging.");
           }

           if (scanAll)
           {
               UpdateFolderRecursive(rootFolder);
           }
           else // Process the files that are listed in feedFile and/or logFile
           {
               bool hasFeedFile = true;
               string feedFile = ConfigurationManager.AppSettings["FeedFile"];
               if (string.IsNullOrEmpty(feedFile) || !File.Exists(feedFile))
               {
                   Console.WriteLine("Miss Feed file!");
                   sw.WriteLine("Miss Feed file!");
                   hasFeedFile = false;
               }
               List<string> feedNameList = null;
               if (hasFeedFile)
               {
                   feedNameList = GetFileNames(feedFile, rootFolder);
                   UpdateFilesAttribute(feedNameList);
               }

               bool hasLogFile = true;
               string logFile = ConfigurationManager.AppSettings["LogFile"];
               if (string.IsNullOrEmpty(logFile) || !File.Exists(logFile))
               {
                   Console.WriteLine("Miss Log file!");
                   sw.WriteLine("Miss LOg file!");
                   hasLogFile = false;
               }
               List<string> logNameList = null;
               if (hasLogFile)
               {
                   logNameList = GetFileNames(logFile, rootFolder);
                   UpdateFilesAttribute(logNameList);
               }
           }
           if (doLog && sw != null)
           {
               sw.Close();
           }
       }

       /// <summary>
       /// Read a file storing all file names that need to be processed
       /// Put all file names into a list
       /// </summary>
       /// <param name="fileName">File Name</param>
       /// <param name="rootFolder">The root folder that holds the file</param>
       private static List<string> GetFileNames(string fileName, string rootFolder)
       {
           List<string> fileNameList = new List<string>();
           if (File.Exists(fileName))
           {
               try
               {
                   using (StreamReader sr = new StreamReader(fileName))
                   {
                       while (sr.Peek() >= 0)
                       {
                           fileNameList.Add(rootFolder + sr.ReadLine());
                       }
                   }
               }
               catch (Exception ex)
               {
                   string logString = DateTime.Now.ToString("yyyy/MM/dd HH:mm") + " - Error in reading file " + fileName + ": " + ex.Message;
                   if (doLog)
                   {
                       sw.WriteLine(logString);
                       sw.Flush();
                   }
                   else
                   {
                       Console.WriteLine(logString);
                   }
               }
           }
           return fileNameList;

       }

       /// <summary>
       /// Go throught all files and remove the read only attribute
       /// </summary>
       /// <param name="fileNames">File names</param>
       private static void UpdateFilesAttribute(List<string> fileNames)
       {
           foreach (string file in fileNames)
           {
               if (File.Exists(file))
               {
                   RemoveFileReadonlyAttribute(file);
               }
               else
               {
                   /*
                   string logString = DateTime.Now.ToString("yyyy/MM/dd HH:mm") + " - Not a valid file: " + file;
                   if (doLog)
                   {
                       sw.WriteLine(logString);
                       sw.Flush();
                   }
                   else
                   {
                       Console.WriteLine(logString);
                   }
                   */
               }
           }
       }

       /// <summary>
       /// Recursively check all files in the folder and its subfolders
       /// Remove read only for all files
       /// </summary>
       /// <param name="path">Folder Name</param>
       private static void UpdateFolderRecursive(string path)
       {
           if (Directory.Exists(path))
           {
               string[] files = Directory.GetFileSystemEntries(path);
               foreach (string entry in files)
               {
                   if (Directory.Exists(entry))
                   {
                       // It's a folder, dig into it's subfolders
                       UpdateFolderRecursive(entry);
                   }
                   else if (File.Exists(entry))
                   {
                       // It's a file, check the attribute
                       RemoveFileReadonlyAttribute(entry);
                   }
               }
           }
       }

       /// <summary>
       /// Check if the file is read only. if so remove the read only attribute.
       /// </summary>
       /// <param name="fileName">file name</param>
       private static void RemoveFileReadonlyAttribute(string fileName)
       {
           if (!File.Exists(fileName))
               return;

           FileInfo fi = new FileInfo(fileName);
           if ((fi.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
           {
               try
               {
                   // Remove the read only attribute
                   fi.Attributes -= FileAttributes.ReadOnly;
                   StringBuilder sb = new StringBuilder();
                   sb.Append(DateTime.Now.ToString("yyyy/MM/dd HH:mm"));
                   sb.Append(" - Read only attribute of file ");
                   sb.Append(fileName.Replace("\\", "/"));
                   sb.Append(" has been removed.");
                   if (doLog)
                   {
                       sw.WriteLine(sb.ToString());
                       sw.Flush();
                   }
                   else
                   {
                       Console.WriteLine(sb.ToString());
                   }
               }
               catch (Exception ex)
               {
                   string logString = DateTime.Now.ToString("yyyy/MM/dd HH:mm") + " - Error in updating " + fileName + " attribute: " + ex.Message;
                   if (doLog)
                   {
                       sw.WriteLine(logString);
                       sw.Flush();
                   }
                   else
                   {
                       Console.WriteLine(logString);
                   }
               }
           }
       }

       /// <summary>
       /// Add File Security for a file
       /// AddFileSecurity(@"c:\Feed\abc.txt", "Everyone", FileSystemRights.FullControl, AccessControlType.Allow);
       /// </summary>
       /// <param name="filep">File Path</param>
       /// <param name="account">Acount for the permission</param>
       /// <param name="right">User right</param>
       /// <param name="controlType">Allow or Deny</param>
       public static void AddFileSecurity(string file, string account, FileSystemRights right, AccessControlType controlType)
       {
           FileSecurity fileSec = File.GetAccessControl(file);
           fileSec.AddAccessRule(new FileSystemAccessRule(account, right, controlType));
           File.SetAccessControl(file, fileSec);
       }
   }
}