CVDtf.cs

#region Using Statements
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;
using Microsoft.Xna.Framework.Graphics.PackedVector;
using System.IO;
using ILNumerics; 

#endregion

namespace visLU.CVD
{
    class CVDtf
    {
        #region Variables

        Texture2D recoloredImage; //recolored Image Ir
        Texture2D recoloredImageTest; //for debug
        Texture2D recoloredImageMask; //selected Pixels only

        List<TransferControlPoint> colorCP; //refrence transfer function Tc color and opacity
        List<TransferControlPoint> alphaCP;

        List<TransferControlPoint> cvdColorCP; //cvd tf Tc* 
        Texture2D cvdTransferTexture;

        int M; //control point count in the reference tf == color labels, default < 8
        int N; //pixel count in the image
        int Nr; //pixel to select from the recolored image, default = 200, Nr > M
        double[] delta; //parameter for color preservation, delta[m] = [0;1], m = 0,...,#(ControlPoints - 1) 
        float lambda;  //parameter for the the tf color adjustmant: lambda = [0;1], labmda = 0 : cvd image == recolored image
        //accumulated weight for the reference tf (= matrixA)
        ILArray<double> omegaWeightsR; 
        ILArray<double> omegaWeightsG;
        ILArray<double> omegaWeightsB;
      
        Point[] selectedPixels; //selcted Pixels positions
        //colors of Nr selected pixels from the recolored image != background pixels
        ILArray<double> recoloredPixelsR;
        ILArray<double> recoloredPixelsG;
        ILArray<double> recoloredPixelsB;
        // = Color[,] b;  //==crk

        //GeneralMatrix x; //== ct*m cvd tf color set
        ILArray<double> xR;
        ILArray<double> xG;
        ILArray<double> xB;

        GraphicsDevice graphics;

        #endregion

        #region Properties
        public float Lambda
        {
            set { lambda = value; }
        }
        public double[] Delta
        {
            set { delta = value; }
        }
        public Texture2D CVDtransferFunction
        {
            get { return cvdTransferTexture; }
        }
        public Point[] SelectedPixels
        {
            get { return selectedPixels; }
        }
        public Texture2D RecoloredImageMask
        {
            get { return recoloredImageMask; }
        }
        #endregion

        #region Constructor
        public CVDtf(Texture2D _recoloredImage, GraphicsDevice _graphics)
        {
            graphics = _graphics;
            recoloredImage = _recoloredImage;
            recoloredImageTest = _recoloredImage;
            recoloredImageMask = new Texture2D(graphics, recoloredImage.Width, recoloredImage.Height, false, SurfaceFormat.Rgba64);
            

            colorCP = GameProperties.Instance.colorControlPoints;
            alphaCP = GameProperties.Instance.alphaControlPoints;

            N = recoloredImage.Width * recoloredImage.Height;
            Nr = 200; //default value; M < Nr <= N!
            M = colorCP.Count;
            if (Nr <= M) Nr = M + (int)((N - M) * 0.5f); //todo optimize this Nr will be tooo big
            if (Nr >= N) Nr = N - (int)((N - M) * 0.5f);
            selectedPixels = new Point[Nr];
            recoloredPixelsR = ILMath.zeros(Nr, 1);
            recoloredPixelsG = ILMath.zeros(Nr, 1);
            recoloredPixelsB = ILMath.zeros(Nr, 1);
            
            //TODO
            delta = new double[M];
            for (int i = 0; i < delta.Length; i++)
            {
                delta[i] = 0.0f;
            }
                 
            lambda = GameProperties.Instance.lambda; //TODO

            //omega Weights 
            omegaWeightsR = ILMath.ones(Nr, M);
            omegaWeightsG = ILMath.ones(Nr, M);
            omegaWeightsB = ILMath.ones(Nr, M);

        }
        #endregion

        #region Methods

        private void saveScreenshot(Texture2D texture, String name, String ext)
        {
            Console.WriteLine("saving " + name);
            try
            {
                if (!name.ElementAt(name.Length - 4).Equals(".")) name = name + "." + ext;
                FileStream stream = File.OpenWrite(name);
                switch (ext)
                {
                    case "png":
                        texture.SaveAsPng(stream, texture.Width, texture.Height);
                        break;
                    case "jpg":
                        texture.SaveAsJpeg(stream, texture.Width, texture.Height);
                        break;
                    default:
                        if (name.ElementAt(name.Length - 4).Equals(".")) name = name.Substring(0, name.Length - 4);
                        name = name + ".png";
                        texture.SaveAsPng(stream, texture.Width, texture.Height);
                        break;
                }

            }
            catch (Exception e)
            {
                Console.WriteLine("Save Screenshot: ");
                Console.WriteLine(e);
            }

        }
        
        public void selectRecoloredPixels()
        {

            saveScreenshot(recoloredImage, "recoloredImageForCVD", "png"); //debug
            Rgba64[] recoloredArray = new Rgba64[recoloredImage.Width * recoloredImage.Height];
            recoloredImage.GetData<Rgba64>(recoloredArray);
            //Color[] recoloredArray = new Color[recoloredImageTest.Width * recoloredImageTest.Height];
            //recoloredImageTest.GetData<Color>(recoloredArray);

            Rgba64[] recoloredArrayMask = new Rgba64[recoloredImage.Width * recoloredImage.Height];
            recoloredArrayMask.Initialize();

            #region debug
            Texture2D selectedPixelsMask  = new Texture2D(graphics, recoloredImage.Width, recoloredImage.Height, false, SurfaceFormat.Color);
            Color[] selectedPixelsArray = new Color[recoloredImage.Width * recoloredImage.Height];
            for (int i = 0; i < selectedPixelsArray.Length; i++)
            {
                selectedPixelsArray[i] = Color.Black;
            }
            #endregion
            for (int j = 0; j < recoloredArrayMask.Length; j++)
            {
                recoloredArrayMask[j] = new Rgba64(0, 0, 0, 1);
            }
            
            Random randomPos = new Random();
            Random randomScale = new Random();
            int next = 0;
            while (next < Nr)
            {
                
                double xScale = randomPos.NextDouble();
                double yScale = randomPos.NextDouble();
                int x = (int)(randomScale.Next(recoloredImage.Width) * xScale);
                int y = (int)(randomScale.Next(recoloredImage.Height) * yScale);
                int selectedPos = x + y * recoloredImage.Width;

                #region median filter
                //get the median
                /*int filtersize = 5;
                List<Vector3> neighbours = new List<Vector3>();

                int offsetX = (-1) * (filtersize - 1) / 2;
                int offsetY = (-1) * (filtersize - 1) / 2;
                for (int j = 0; j < filtersize; j++)
                {
                    offsetX = (-1) * (filtersize - 1) / 2;
                    for (int i = 0; i < filtersize; i++)
                    {
                        //rand
                        int nX = (int)MathHelper.Clamp(x + offsetX, 0, recoloredImage.Width - 1);
                        int nY = (int)MathHelper.Clamp(y + offsetY, 0, recoloredImage.Height - 1);
                        Vector4 neighbour = recoloredArray[nX + nY*recoloredImage.Width].ToVector4();
                        neighbours.Add(new Vector3(neighbour.X, neighbour.Y, neighbour.Z));

                        offsetX++;
                    }
                    offsetY++;
                }

                //sort list 
                neighbours.Sort(delegate(Vector3 n1, Vector3 n2) { return (n1.Length().CompareTo(n2.Length())); });

                Vector3 selectedPixel = neighbours[(filtersize * filtersize - 1) / 2];*/
                #endregion

                #region average filter
                /*int filtersize = 11;
                Vector3 avgColor = Vector3.Zero;

                int offsetX = (-1) * (filtersize - 1) / 2;
                int offsetY = (-1) * (filtersize - 1) / 2;
                for (int j = 0; j < filtersize; j++)
                {
                    offsetX = (-1) * (filtersize - 1) / 2;
                    for (int i = 0; i < filtersize; i++)
                    {
                        //rand
                        int nX = (int)MathHelper.Clamp(x + offsetX, 0, recoloredImage.Width - 1);
                        int nY = (int)MathHelper.Clamp(y + offsetY, 0, recoloredImage.Height - 1);
                        Vector4 neighbour = recoloredArray[nX + nY * recoloredImage.Width].ToVector4();
                        avgColor.X += neighbour.X;
                        avgColor.Y += neighbour.Y;
                        avgColor.Z += neighbour.Z;
                        offsetX++;
                    }
                    offsetY++;
                }

                Vector3 selectedPixel = new Vector3(avgColor.X / (filtersize * filtersize), avgColor.Y / (filtersize * filtersize), avgColor.Z / (filtersize * filtersize)); 
                 */
                #endregion

                Vector4 selectedPixel = recoloredArray[selectedPos].ToVector4();
                if (selectedPixel.X > 0.2f || selectedPixel.Y > 0.2f || selectedPixel.Z > 0.2f)  //ignore background pixels
                {

                  
                    selectedPixels[next] = new Point(x, y);
                    recoloredPixelsR.SetValue((double)selectedPixel.X, next, 0);
                    recoloredPixelsG.SetValue((double)selectedPixel.Y, next, 0);
                    recoloredPixelsB.SetValue((double)selectedPixel.Z, next, 0);
                    //Console.WriteLine("Weights for SP:" + selectedPixelsInArray[i] + ":" + selectedPixel.X + "," + selectedPixel.Y + "," + selectedPixel.Z + "," + selectedPixel.W);
                    //Console.WriteLine("Selected Pixels: x" + x + ", y:" + y + ", pos:" + (x + y * recoloredImage.Width));
                    //Console.WriteLine("Selected Colors: R" + selectedColor.R + ", G:" + selectedColor.G + ", B:" + selectedColor.B);
                    recoloredArrayMask[selectedPos] = new Rgba64((float)x / recoloredImage.Width, (float)y / recoloredImage.Height, 0.0f, 1.0f);
                    selectedPixelsArray[selectedPos] = new Color(selectedPixel); //debug
                    next++;
                }
            }
             
          
            #region debug
            //Console.WriteLine("#Selected Colors:" + selectedPixelsPos.Length); 
            recoloredImageMask.SetData<Rgba64>(recoloredArrayMask);
            selectedPixelsMask.SetData<Color>(selectedPixelsArray);
            if (GameProperties.Instance.debugOutput)
            {
                saveScreenshot(recoloredImageMask, "recoloredImageMask", "png");
                saveScreenshot(selectedPixelsMask, "selectedPixelsMask", "png");
            }
            //selectedPixelsMask.Dispose();
            #endregion

        }


        public Texture2D calculateCVDColors()
        {
           
            #region calculate cvd colors for the tf
            xR = ILMath.linsolve(omegaWeightsR, recoloredPixelsR);
            xG = ILMath.linsolve(omegaWeightsG, recoloredPixelsG);
            xB = ILMath.linsolve(omegaWeightsB, recoloredPixelsB);

            //check variance and error
            //E1
            ILArray<double> bTestR = ILMath.multiply(omegaWeightsR, xR);
            ILArray<double> errR = ILMath.abs(bTestR - recoloredPixelsR);
            ILArray<double> errRnorm = ILMath.norm(bTestR - recoloredPixelsR);

            ILArray<double> bTestG = ILMath.multiply(omegaWeightsG, xG);
            ILArray<double> errG = ILMath.abs(bTestG - recoloredPixelsG);
            ILArray<double> errGnorm = ILMath.norm(bTestG - recoloredPixelsG);

            ILArray<double> bTestB = ILMath.multiply(omegaWeightsB, xB);
            ILArray<double> errB = ILMath.abs(bTestB - recoloredPixelsB);
            ILArray<double> errBnorm = ILMath.norm(bTestB - recoloredPixelsB);

            #region debug
            //scale 
            double max = 0.0f;
            for (int i = 0; i < M; i++)
            {
                max = Math.Max(Math.Abs(xR.GetValue(i, 0)), max);
                max = Math.Max(Math.Abs(xG.GetValue(i, 0)), max);
                max = Math.Max(Math.Abs(xB.GetValue(i, 0)), max);
 
           }

            /*for (int i = 0; i < M; i++)
            {
                xR.SetValue((xR.GetValue(i, 0)/max) / 2 + 0.5f ,i);
                xG.SetValue((xG.GetValue(i, 0)/max) / 2 + 0.5f, i);
                xB.SetValue((xB.GetValue(i, 0)/max) / 2 + 0.5f, i);
            }*/
            #endregion

            //cvd transfer function <-> reference transfer function
            // sort reference ControlPoints Ct by isoValue
            List<TransferControlPoint> ctColorCP = GameProperties.Instance.colorControlPoints;
            ctColorCP.Sort(delegate(TransferControlPoint cp1,
                TransferControlPoint cp2) { return cp1.isoValue.CompareTo(cp2.isoValue); });

            ILArray<double> deltaWeights = ILMath.zeros(M, 1);
            ILArray<double> referenceTFR = ILMath.zeros(M, 1);
            ILArray<double> referenceTFG = ILMath.zeros(M, 1);
            ILArray<double> referenceTFB = ILMath.zeros(M, 1);
            for (int i = 0; i < M; i++)
            {
                deltaWeights.SetValue(delta[i], i, 0);
                referenceTFR.SetValue(ctColorCP[i].color.X * delta[i], i, 0);
                referenceTFG.SetValue(ctColorCP[i].color.Y * delta[i], i, 0);
                referenceTFB.SetValue(ctColorCP[i].color.Z * delta[i], i, 0);
            }

            ILArray<double> xR2 = ILMath.linsolve(deltaWeights, referenceTFR);
            ILArray<double> xG2 = ILMath.linsolve(deltaWeights, referenceTFG);
            ILArray<double> xB2 = ILMath.linsolve(deltaWeights, referenceTFB);

            //E1
            ILArray<double> bTestR2 = ILMath.multiply(deltaWeights, xR2);
            ILArray<double> errR2 = ILMath.abs(bTestR2 - referenceTFR);
            ILArray<double> errRnorm2 = ILMath.norm(errR2);

            ILArray<double> bTestG2 = ILMath.multiply(deltaWeights, xG2);
            ILArray<double> errG2 = ILMath.abs(bTestG2 - referenceTFG);
            ILArray<double> errGnorm2 = ILMath.norm(errG2);

            ILArray<double> bTestB2 = ILMath.multiply(deltaWeights, xB2);
            ILArray<double> errB2 = ILMath.abs(bTestB2 - referenceTFG);
            ILArray<double> errBnorm2 = ILMath.norm(errB2);

          

            //lambda
            for (int i = 0; i < M; i++)
            {
              xR.SetValue((1 - lambda) * xR.GetValue(i, 0) + lambda * xR2.GetValue(0,0) , i, 0);
              xG.SetValue((1 - lambda) * xG.GetValue(i, 0) + lambda * xG2.GetValue(0, 0), i, 0);
              xB.SetValue((1 - lambda) * xB.GetValue(i, 0) + lambda * xB2.GetValue(0, 0), i, 0);
             }
       
            #endregion

            #region set in the cvd cp list
            cvdColorCP = new List<TransferControlPoint>();
            for (int i = 0; i < M; i++)
            {
                //cvdColorCP.Add(new TransferControlPoint((float)xR.GetValue(M - 1 - i, 0), (float)xG.GetValue(M - 1 - i, 0), (float)xB.GetValue(M - 1 - i, 0), ctColorCP[i].isoValue));
                cvdColorCP.Add(new TransferControlPoint((float)xR.GetValue(i, 0), (float)xG.GetValue(i, 0), (float)xB.GetValue(i, 0), ctColorCP[i].isoValue));
            }
            GameProperties.Instance.cvdColorControlPoints = cvdColorCP;
            #endregion


            #region interpolate between ControlPoints
            float amount = 0.0f;
            Color[] colorArray = new Color[256];

            // sort ControlPoints by isoValue
            cvdColorCP.Sort(delegate(TransferControlPoint p1,
                TransferControlPoint p2) { return p1.isoValue.CompareTo(p2.isoValue); });

            // before first ControlPoint: fill with black and alpha 0.0f
            for (int i = 0; i < cvdColorCP[0].isoValue; i++)
            {
                colorArray[i] = new Color(0.0f, 0.0f, 0.0f, 0.0f);
            }

            for (int j = 0; j < cvdColorCP.Count - 1; j++)
            {
                for (int i = cvdColorCP[j].isoValue; i <= cvdColorCP[j + 1].isoValue; i++)
                {
                    amount = (((float)i - cvdColorCP[j].isoValue) /
                        (cvdColorCP[j + 1].isoValue - cvdColorCP[j].isoValue));
                    colorArray[i] = Color.Lerp(new Color(cvdColorCP[j].color),
                    new Color(cvdColorCP[j + 1].color), amount);
                }
            }

            //after last ControlPoint: fill with black and alpha
            for (int i = cvdColorCP[cvdColorCP.Count - 1].isoValue + 1; i < 256; i++)
            {
                colorArray[i] = new Color(0.0f, 0.0f, 0.0f, 0.0f);
            }
            #endregion

            #region set texture
            //interpolate texture
            cvdTransferTexture = new Texture2D(graphics, 256, 1, false, SurfaceFormat.Color);
            cvdTransferTexture.SetData<Color>(colorArray);
            saveScreenshot(cvdTransferTexture, "CVDTF", "png"); 
            #endregion

            return cvdTransferTexture;
        }


        public void setTFWeights(Texture2D weights, int controlPointIdx)
        {
            if (controlPointIdx >= M) return;

            Rgba64[] weightsArray = new Rgba64[weights.Width * weights.Height];
            weights.GetData<Rgba64>(weightsArray);
           
      
            for (int i = 0; i < selectedPixels.Length; i++)
            {
                int pos = (selectedPixels[i].X + selectedPixels[i].Y * weights.Width);
                Vector4 nextWeight = weightsArray[pos].ToVector4();

                omegaWeightsR.SetValue((double)nextWeight.X, i, controlPointIdx);
                omegaWeightsG.SetValue((double)nextWeight.Y, i, controlPointIdx);
                omegaWeightsB.SetValue((double)nextWeight.Z, i, controlPointIdx);

                if (GameProperties.Instance.debugOutput)
                {
                    Console.WriteLine("Weights for CP:" + controlPointIdx + ":" + nextWeight.X + "," + nextWeight.Y + "," + nextWeight.Z + "," + nextWeight.W);
                }
                
            }

        }
        #endregion
    }
}