Instant VM restore

This script is used to instantly restore a virtual machine from a netapp flexcloned volume

###################################################
#$filercreds is defined in profile.ps1 if you dont have it in your profile.ps1 - uncomment the following
##Run this part once
#read-host -AsSecureString | ConvertFrom-SecureString | out-file c:\filerpassword.txt

##Add this part to your profile.ps1
#$filerpassword = cat C:\filerpassword.txt | ConvertTo-SecureString
#$Filercreds = New-Object -TypeName System.Management.Automation.PScredential -argumentlist "admin" , $filerpassword
#################################################


Connect-VIServer vserver1,vserver2
$RecoveryVM = read-host "Enter the name of the VM that you want to do an instant restore"

$sanityCheck = read-host "By proceeding you will be removing" $RecoveryVM "from the invenotry and then mounting and running the VM from a flexclone - the script will then migrate the VM from the flexclone and overwrite the old production volume and dismount the flexclone - Proceed (Y/N)"
If ($sanityCheck -like "*N*")
    {write-host "exiting"
     pause
     exit }

$recoveryVM = get-vm | Where-Object {$_.name -like $RecoveryVM}
$recoveryhost = get-vm $recoveryvm | get-vmhost
$RecoveryVMParentStorage = get-vm $RecoveryVM | get-datastore | Where-object {($_.name -notlike "*swap*") -and ($_.name -notlike "*srm*") -and ($_.name -notlike "*page*")}

If ($RecoveryVMParentStorage.count -gt 1)
    { Write-host "Attached disks span more than 1 netapp volume - you'll have to do a manual restore using SMVI"
      Pause
      exit
    }

get-vm $recoveryvm | Shutdown-VMGuest -Confirm:$false

$i = 0
do{
    $i++
    Write-host "waiting for shutdown to complete"
    sleep 1
    $powerstate = get-vm $RecoveryVM
    if ($i -gt 120){
    write-host "VM is not shutting down gracefully - performing power off"
      get-vm $RecoveryVM | stop-vm -Confirm:$false
                    }
    }until($powerstate.powerstate -like "*off*")


remove-vm $recoveryvm -Confirm:$false  

# Connect to the appropriate filer
            if ($RecoveryVMParentStorage -like "*filera*"){
                 Connect-NcController namefilera -Credential $Filercreds -Vserver filera1              
                }
            if ($RecoveryVMParentStorage -like "*filerb*"){
                 Connect-NcController namefilerb -Credential $Filercreds -Vserver filerb1                
                }

$NetappVols = get-ncvol $RecoveryVMParentStorage.name

foreach ($netappvol in $NetappVols){
$AllNetavols = get-ncvol $netappvol | Get-NcSnapshot
}


# Create a list box containing all snapshots from the volume - Partially plagerised from https://technet.microsoft.com/en-us/library/ff730949.aspx

[void] [System.Reflection.Assembly]::LoadWithPartialName("System.Windows.Forms")
[void] [System.Reflection.Assembly]::LoadWithPartialName("System.Drawing")

$objForm = New-Object System.Windows.Forms.Form
$objForm.Text = "Select a snapshot to clone"
$objForm.Size = New-Object System.Drawing.Size(300,200)
$objForm.StartPosition = "CenterScreen"

$objForm.KeyPreview = $True
$objForm.Add_KeyDown({if ($_.KeyCode -eq "Enter")
    {$x=$objListBox.SelectedItem;$objForm.Close()}})
$objForm.Add_KeyDown({if ($_.KeyCode -eq "Escape")
    {$objForm.Close()}})

$OKButton = New-Object System.Windows.Forms.Button
$OKButton.Location = New-Object System.Drawing.Size(75,120)
$OKButton.Size = New-Object System.Drawing.Size(75,23)
$OKButton.Text = "OK"
$OKButton.Add_Click({$x=$objListBox.SelectedItem;$objForm.Close()})
$objForm.Controls.Add($OKButton)

$CancelButton = New-Object System.Windows.Forms.Button
$CancelButton.Location = New-Object System.Drawing.Size(150,120)
$CancelButton.Size = New-Object System.Drawing.Size(75,23)
$CancelButton.Text = "Cancel"
$CancelButton.Add_Click({$objForm.Close()})
$objForm.Controls.Add($CancelButton)

$objLabel = New-Object System.Windows.Forms.Label
$objLabel.Location = New-Object System.Drawing.Size(10,20)
$objLabel.Size = New-Object System.Drawing.Size(280,20)
$objLabel.Text = "Available Snapshots for $NetappVols.name :"
$objForm.Controls.Add($objLabel)

$objListBox = New-Object System.Windows.Forms.ListBox
$objListBox.Location = New-Object System.Drawing.Size(10,40)
$objListBox.Size = New-Object System.Drawing.Size(260,20)
$objListBox.Height = 80


foreach ($AllNetavol in $AllNetavols){
[void] $objListBox.Items.Add($AllNetavol.name )
}

$objForm.Controls.Add($objListBox)

$objForm.Topmost = $True

$objForm.Add_Shown({$objForm.Activate()})
[void] $objForm.ShowDialog()



# get the flexclone as selected by the user
$Flexclone = get-ncvol $netappvol | get-ncsnapshot | Where-object {$_.name -like $objListBox.SelectedItem}

$FlexcloneVolname = $netappvol.name+"FC_InstantRestore"

#Check if the volume is already present - if it is warn the user and exit
if (Get-Datastore | Where-Object {$_.name -like $FlexcloneVolname}) {
        Clear
        write-host "*** Error*** The volume" $FlexcloneVolname "already exists - check there are no VMs running on it , unmount it and rerun the script"
        Pause
        exit
}
   

# spawn the flexcloned volume from the snapshot
get-ncvol $netappvol | New-NcVolClone -CloneVolume $FlexcloneVolname -ParentSnapshot $Flexclone

# figure out which host to attach the export to , Create the export , attach the export
$recoveryhostIP = $recoveryhost | Get-VMHostNetworkAdapter
$exportIPS = $recoveryhostIP.ip | Where-Object {$_ -cmatch $RecoveryVMParentStorage.remotehost.Split(".")[2]}
Get-Ncvol $FlexcloneVolname | Mount-NcVol -JunctionPath "/$FlexcloneVolname"
Add-NcNfsExport -Path /$FlexcloneVolname -ReadWrite @($exportIPS) -Root @($exportIPS)
Sleep 30
New-Datastore -VMHost $recoveryhost -Name $FlexcloneVolname  -Path /$FlexcloneVolname -NfsHost $RecoveryVMParentStorage.remotehost -Nfs

# Set up Search for .VMX Files in Datastore (partially plagerised from http://www.wooditwork.com/2011/08/11/adding-vmx-files-to-vcenter-inventory-with-powercli-gets-even-easier/)
   $ds = Get-Datastore -Name $FlexcloneVolname | %{Get-View $_.Id}
   $SearchSpec = New-Object VMware.Vim.HostDatastoreBrowserSearchSpec
   $SearchSpec.matchpattern = "*.vmx"
   $dsBrowser = Get-View $ds.browser
   $DatastorePath = "[" + $ds.Summary.Name + "]"

   # Find all .VMX file paths in Datastore, filtering out ones with .snapshot (Useful for NetApp NFS)
   $VMXfile = $dsBrowser.SearchDatastoreSubFolders($DatastorePath, $SearchSpec) | where {($_.FolderPath -notmatch ".snapshot") -and ($_.folderpath -cmatch $RecoveryVM.name)} | %{$_.FolderPath + ($_.File | select Path).Path}

   #Register the .vmx Files
   New-VM -VMFilePath $VMXFile -VMHost $recoveryhost -Location $recoveryVM.Folder -ResourcePool $RecoveryVM.ResourcePool -RunAsync
 

#Wait until the VM has been registered on the vcentre server
            do {
                Write-host "confirming that " $recoveryvm "is registered on vcentre server before continuing -"
                $VMwaiter = get-vm $recoveryvm -erroraction silentlycontinue
                Sleep 5
                }
                until ($VMwaiter)

# start the VM up and answer the VM startup question
        get-vm $RecoveryVM | start-vm  -Confirm:$false -ErrorAction SilentlyContinue
            Sleep 20
        get-vm $RecoveryVM | Get-VMQuestion | Set-VMQuestion -Option button.uuid.copiedTheVM -Confirm:$false -ErrorAction SilentlyContinue



Write-host "VM has been started on flexclone Volume - beginning Storage vmotion to" $recoveryVMparentstorage " you can begin using the VM now"

#svmotion the VM back to the original location
  get-vm $RecoveryVM | move-vm -Datastore $RecoveryVMParentStorage -DiskStorageFormat Thin -VMotionPriority High
 


#Wait for the VM to complete svmotion
    do {
        $VMwaiter = get-vm $recoveryvm | get-datastore | Where-object {($_.name -notlike "*swap*") -and ($_.name -notlike "*srm*")}
        Write-host  "waiting for VM to complete svmotioning"
        sleep 10
        }
        until ($vmwaiter -like $RecoveryVMParentStorage)


#Trigger a SIS on the original volume
start-ncsis -path $netappvol -scan

#cleanup + remove recovery volume from ESXi Host and export & Volume from filer
Remove-Datastore -VMHost $recoveryhost -Datastore $FlexcloneVolname -Confirm:$false
#Determine the export policy Name - if its default then abort
$exportpolicyName = Get-NcVol $FlexcloneVolname |  select @{n="PolicyName"; E={ $_.VolumeExportAttributes.policy}}
 If ($exportpolicyName.policyname -notlike "*Default*"){
  remove-ncnfsexport -paths /$FlexcloneVolname -Confirm:$false
  Dismount-NcVol -Name $FlexcloneVolname -Confirm:$false
  set-ncvol $FlexcloneVolname -Offline -Confirm:$false
  Remove-NcVol $FlexcloneVolname -confirm:$false
  get-ncexportpolicy | Where-object {$_.PolicyName -like $exportpolicyname.policyname} | Remove-NcExportPolicy -Confirm:$false
    }

 Else{
 write-host "Trying to modify Default policy - Aborting"
 Pause
 }

Find the ILO IP of a HP ESXi Server through the CLI

Find the ILO IP of a HP ESXi Server through the CLI

cd /opt/hp/tools

./hponcfg -w /tmp/ilo_config.txt

Open /tmp/ilo_config.txt

Neatened version of snapshot reporting scrip 

import-module dataontap
$HourlyResultsTooMany= @()
$HourlyResultsTooFew= @()
$HourlyResultsNone= @()

$filers = "filer1,filer2,filer3,filer4,filer5,filer6"
$filers | foreach-object {
    #write-host $_
    connect-nacontroller $_
    $vmvols = get-NaVol | Where-Object { ($_.name -like "*vmnfs*" -or $_.name -like "*vmwaredata*") -and -not ($_.name -like "*m" -or $_.name -like "*m2" -or $_.name -like "*page" -or $_.name -like "*srmph"  -or $_.name -like "*swap" -or $_.name -like "*sw" -or $_.name -like "*vdi*" -or $_.name -like "*NonProd" -or $_.name -like "*filer1syslog"  -or $_.name -like "*filer2vdisys"  -or $_.name -like "*filer40*") }
    $vmvolc = $vmvols | measure-object
    if ($vmvolc.count -gt 0) {
        $vmvols | ForEach-Object {
            $vmsnaps = Get-NaSnapshot $_.name | Where-Object { ($_.name -like "smvi*") -and ($_.name -like "*hourly*") -and ($_.name -notlike "*appsercvr*")} | measure-object
            write-host "volume: " $_.name " Snapshots: " $vmsnaps.count
             if ($vmsnaps.Count -eq 0) {
                   
                      $HourlyResultsNone += New-Object psobject -Property @{
                                           
                                            'Volume' = $_.name
                   
                                       }

                    } 
             if ($vmsnaps.Count -ge 25 ) {
                   
                     $HourlyResultsTooMany += New-Object psobject -Property @{
                            
                                            'Volume' = $_.name
                                            'No Of Snapshots' = $vmsnaps.count
                                            }                   
                       }
             if ($vmsnaps.Count -lt 24 ) {
                   
                     $HourlyResultsTooFew += New-Object psobject -Property @{
                            
                                            'Volume' = $_.name
                                            'No Of Snapshots' = $vmsnaps.count
                                        }
                    }
        }
    }
}
If ( $DailyResultsNone.count -ge 0) {
                    Send-MailMessage -From "someone@companyname.ie" -To "someone@companyname.ie" -Subject 'Hourly Snapshot Alert , Some Volumes do not have any Hourly snapshots' -Body ( $HourlyResultsNone | Out-String )
                     }

If ( $DailyResultsTooFew.count -ge 0) {
                     Send-MailMessage -From "someone@companyname.ie" -To "someone@companyname.ie" -Subject 'Hourly Snapshot Alert , Some Volumes do not have enough Hourly snapshots' -Body ( $HourlyResultsTooFew | Out-String )
                     }
                   
If ( $DailyResultsTooMany.count -ge 0) {
                     Send-MailMessage -From "someone@companyname.ie" -To "someone@companyname.ie" -Subject 'Hourly Snapshot Alert , Some Volumes have too many Hourly snapshots' -Body ( $HourlyResultsTooMany | Out-String )
                     }

Find volumes without autodelete or autosize enabled

powershell script to loop through each of the filers looking for any VMnfs volume which did not have the autosize enabled and was not able to autogrow


import-module dataontap
$results= @()
$filers = "Filer1","Filer2","Filer3","Filer4","Filer5","Filer6"
$filers | foreach-object {
    write-host $_
    connect-nacontroller $_
    $vmvols = get-NaVol | Where-Object { ($_.name -like "*vmnfs*" -or $_.name -like "*vmwaredata*" -or $_.name -like "*VDI*" ) -and -not ($_.name -like "*m" -or $_.name -like "*m2" -or $_.name -like "*page" -or $_.name -like "*srmph"  -or $_.name -like "*swap" -or $_.name -like "*sw" -or $_.name -like "*vdi*" -or $_.name -like "*NonProd" -or $_.name -like "*filer1vmnfsvolsyslog"  -or $_.name -like "*filer2vmnfsvolvdisys"  -or $_.name -like "*filer3vmnfsvol0*") }
    foreach ($volume in $vmvols) {
        $VolumeName=$volume.name
        $volumeSize=$volume.SizeTotal
        $VolumeSizeAvailable=$volume.SizeAvailable
        $autogrow = Get-NaVolAutosize -name $volume
        $AutogrowEnabled = $autogrow.isenabled
        $SnapshotAutodelete = Get-nasnapshotautodelete $volume
        }

        If  (($AutogrowEnabled -notlike "true" ) -or ($SnapshotAutodelete[4].OptionValue -notlike "on") )
        {

       
        $results += New-Object psobject -Property @{
                                       
                                        'Volume' = $volume
                                        'AutogrowEnabled' = $autogrowEnabled
                                        'SnapshotAutodelete' = $SnapshotAutodelete[4].OptionValue
                                       }
       }
}

WMI Filter for applying GPO to all instances of internet explorer where the version is less than 9.0

Create WMI filter -
SELECT * FROM CIM_DataFile WHERE Filename = 'iexplore' AND Path = '\\Program Files\\Internet Explorer\\' AND version <'9.0'

Powershell script to age out Netapp snapshots

I had a an incident where i had to recreate the SMVI jobs on one of our Filers , we have a requirement to keep 26 days of backups online.
As always space was at a premium so What i wanted to do was to delete the old version of the SMVI
Snapshot each time after the new snapshot was taken...

so i created a script which loops through decreasing the value of the number of backups we want to keep

$days = -26
for ($loops = 26;$loops -ge 0; $loops --)
 {
  $BackupstoKeep = (get-date).adddays($days)
  Connect-nacontroller "Filer1"
  Get-Navol | Get-NaSnapshot | where-object { $_.name -like "smvi_*" -and $_.created -lt $Backupstokeep }  | remove-nasnapshot
  Connect-nacontroller "Filer2"
  Get-Navol | Get-NaSnapshot | where-object { $_.name -like "smvi_*" -and $_.created -lt $Backupstokeep }  | remove-nasnapshot
  $days++
 
Start-sleep 86400
}

Enable SSH on all hosts and set the policy to automatic

(in a non production lab obviously)

First , have a look at the current state of the SSH service on all of the hosts

Get-VMHost | Get-VMHostService | Where {$_.label -like "SSH"} | select VMhost , running , policy , Label

Then start the SSH service on each of your hosts 

Get-VMHost | Get-VMHostService | Where {$_.label -like "SSH"} | Start-VMHostService

Then ensure that it starts up each time on startup

Get-VMHost | Get-VMHostService | Where {$_.label -like "SSH"} | Set-VMHostService -Policy Automatic

Migration of Netapp Filer in one geographic location whilst maintaining large snapmirror relationship with original destination.

I was upgrading a filer in one of our remote sites , part of the project called for the migration of our CIFS data , the CIFS volume contained over 1 TB of data and our link back to our Primary datacenter was 20Mbps. Snapping all of this data back down would have been a time consuming excersize
and needless waste of resources.

What follows are the high level steps i took to ensure that the new source filer replaced the old source filer in the snapmirror relation ship

Filer A = Old Regional Filer
Filer B = New Regional Filer
Filer C = Primary Datacentre Filer


1. Snapmirror relationship between FilerA(FilerAcifsvol1) already exists to FilerC(FilerAcifsvol1M)
2. Create Snapmirror relationship between FilerA(FilerAcifsvol1) and FilerB(FilerBcifsvol1)
3. Allow all Snapmirrors to become consistent
4. Change Snapmirror trigger to manual
5. Stop CIFS Service on FilerA
6. Perform final snapmirror in the following sequence
    FilerA(FilerAcifsvol1) ->FilerC(FilerAcifsvol1M)
    FilerA(FilerAcifsvol1) ->FilerB(FilerBcifsvol1)
   
Make sure that the correct snapshots exist on each filer
7. Quiesce and break  FilerA(FilerAcifsvol1) ->FilerB(FilerBcifsvol1) snapmirror relationship
8. Modify \\filerC\c$\etc\snapmirror.conf on FilerC as follows
    replace
   FilerA:FilerAcifsvol1 FilerC:FilerAcifsvol1M KBS=XXX ... ... ... ...
    With
  FilerB:FilerBcifsvol1 FilerC:FilerAcifsvol1M KBS=XXX ... ... ... ...

9. Putty  into filerC and execute the following command
   snapmirror update -S FilerB:FilerBcifsvol1 -w FilerC:FilerAcifsvol1M

Powershell script to create a VLAN'd portgroup on vswitch0 for each ESX host in the appropriate cluster.


Connect-VIServer vcentreServer
$DublinSite1ProdHosts = get-vmhost | where-object {($_.parent -like "DublinCluster01")}
$DublinSite1ProdHosts | ForEach-Object {( get-vmhost $_.name ) | Get-VirtualSwitch -name "vswitch0" | New-VirtualPortGroup -Name Site1IsolatedObsoleteServers -VLanId 181 }


Connect-VIServer vcentreServer2
$DublinSite2ProdHosts = get-vmhost | where-object {($_.parent -like "DublinCluster02")}
$DublinSite2ProdHosts | ForEach-Object {( get-vmhost $_.name ) | Get-VirtualSwitch -name "vswitch0" | New-VirtualPortGroup -Name Site2IsolatedObsoleteServers -VLanId 182 }

PowerShell script to apply a specific GPO and disable another Specific GPO .

$ohewes = Get-ADOrganizationalUnit -LDAPFilter '(name=*)' -searchbase 'ou=users,ou=users,ou=users,dc=ie,dc=companyname,dc=companyname,dc=com' -searchscope Onelevel | Where-object {$_.name -notlike "IT" -and $_.name -notlike "Unassigned*"}

$ohewes | ForEach-Object {
    New-GPLink -name "EncryptionCheck - UserPolicy" -target $_.DistinguishedName  -linkEnabled Yes
}

$ohewes | ForEach-Object { set-GPLink -name "OldEncryptionCheck - User Policy" -target $_.DistinguishedName  -linkEnabled no -whatif }

PowerShell script to refresh training environment to most recent snapshot

#Refreshes the training system to the latest snapshot
connect-viserver viserver
$PSEmailserver = "SMTP.companyname.com"
$CurrentTrainingSnap = (get-snapshot Appserver** | sort-object -Property Created -Descending | select-object -ExpandProperty Name -First 1)
Set-VM Appserver** -Snapshot $CurrentTrainingSnap -Confirm:$false
Send-MailMessage -From "emailaddress" -To "emailaddress" -cc "emailaddress" -Subject "Appserver** has been refreshed"

PowerShell script to check for unencrypted computers and send email alert

Computers log any decryption event to a log file , if the log file is older than a day
then the process is assumed to have failed , in this case an email is sent to our helpdesk system which automatically logs a call for the helpdesk staff to contact the end user and manually remediate the device

$PSEmailserver = "SMTP.emailserver.com"
$DateToCompare = (Get-date).addDays(-1)
$PathToCheck = "\\fileserver\emcryptionuninstall$\DecryptLimbo\"
$LimboComputers = get-childitem $PathToCheck | Where-object {$_.lastwritetime -lt $DatetoCompare}
$limbocomputers | ForEach-Object { Send-MailMessage -From "Security@company.ie" -To "me@company.ie" -cc "others@company.ie" -Subject "The following computer has not completed the encryption Process" -Body (get-content $_.fullname)}

Unable to power off a VM , Hangs at 95%

I had a problem with a Dev Machine that had become completely unresponsive , we were unable to ping it , and unable to control it using vsphere.
I tried to power it down but the progress halted at 95% ,
I putty'd in to the host and tried to power it down using
vim-cmd vmsvc/getallvms and getting the  vmid
then
vim-cmd vmsvc/power.off XX i then checked the power state
vim-cmd vmsvc/power.getsate XX and found that the machine was still up

I then tried to pull the rug from under the VM by running

esxcli vm process list - obtained the worldID of the misbehaving VM
esxcli vm process kill --type=soft --world-id=XXXX
Checked powerstate with 
vim-cmd vmsvc/power.getsate XX - failed tried a hard kill

esxcli vm process kill --type=hard --world-id=XXXX-Failed again , so last  chance saloon

esxcli vm process kill --type=force --world-id=XXXX
 

Netapp Powershell command to find all snapshots older than 28 days

1 Download dataontap PS module and extract to c:\windows\system32\windowspowershell\1.0\modules

2 Import the netapp powershell module
   Import-Module dataontap

3 define $28days as being todays date - 28 days
   $28days = (get-date).adddays(-28)

4 connect to the controller
   connect-nacontroller "controllername"

5 view the list of snapshots that are older than 28 days
   Get-Navol | Get-NaSnapshot | where-object { $_.name -like "smvi_*" -and $_.created -lt $28days }

6 See what snapshots will be deleted
   Get-Navol | Get-NaSnapshot | where-object { $_.name -like "smvi_*" -and $_.created -lt $28days }  | remove-nasnapshot -whatif

7 Delete the snapshots
     Get-Navol | Get-NaSnapshot | where-object { $_.name -like "smvi_*" -and $_.created -lt $28days }  | remove-nasnapsho

VMware Site Recovery Manager 5  & Netapp SRA 2.0 Failure

(Unable to export the NAS device Ensure that the correct export rules are specified in the ontap_config.txt file  )

VMware Site Recovery Manager and SRA installed and configured , Sites and Resource mappings configured correctly , Netapp Protected site volumes Were mirroring correctly , SRA 2.0 array manager installed and configured correctly , Protection groups defined and configured correctly.

When i attempted to run a recovery plan for any of our sites , i would receive an error on the “recovery steps” tab which stated “Error - Failed to recover datastore 'Vol1'. Failed to create snapshots of replica devices. Failed to create snapshot of replica device /vol/Vol1m. SRA command 'testFailoverStart' failed for device '/vol/Vol1m'. Unable to export the NAS device Ensure that the correct export rules are specified in the ontap_config.txt file  

I Checked the content ontap_config.txt file , this file is used to define the R/W and Root hosts for accessing the cloned export of the mirrored production volume , i confirmed that the VMkernel IP’s for the NFS VMkernel were listed. 

I reset and reran the SRM test and examined the VMware-DR-XXX.log file  
Here i could see that the cloned export of the paging volume came online 

“--> 07-08-2012T10:34:35  Export /vol/testfailoverClone_nss_v10745371_volpagem has root & r/w IP=10.10.10.1”

But the production volumes failed

--> 07-08-2012T10:34:35  Checking existence of storage device /vol/Vol1m
--> 07-08-2012T10:34:35  Storage device /vol/Vol1m is a NFS export
--> 07-08-2012T10:34:35  Creating test Clone volume testfailoverClone_nss_v10745371_Vol1m
--> 07-08-2012T10:34:59  Mapping Export /vol/testfailoverClone_nss_v10745371_Vol1m
--> 07-08-2012T10:34:59  Modify the exportfs for path /vol/testfailoverClone_nss_v10745371_volm2
--> 07-08-2012T10:34:59  Modify failed with error: No such file or directory  

 I logged on to the recovery site filer and had a look at the exports file ,
In there i could see that volpagem had the correct VMkernel IP’s listed as R/W and Root hosts              but for the other two production volumes , the default “All hosts” was listed for both R/W and Root hosts , after much searching i found this post :  

http://communities.vmware.com/message/2051567

The key point being  
“This error is caused by a flaw in the NetApp SRA 2.0. If you have an "-actual" statement in the /etc/exports file on the snapmirror destination filer the SRA will fail to create the flexvol-sharename. So if you are carefull and only use sharenames that equals volumenames for all shares (!) then you avoid the "-actual" statement and the SRA seems to work.

NetApp has confirmed this to be a bug in the SRA.

I read this to mean that if the production export contains  –actual it will cause it to fail , i confirmed that none of the production exports were actual path exports , I thought i hit a brick wall until a colleague noticed that one of the unrelated exports contained an “-actual statement” i confirmed the export was not in use and removed it.
I reran the test and it succeeded.

Powershell script to alert for missing snapshots

We had a problem with SMVI not taking backups and to make matters worse not alerting us to the fact that it was not taking backups , the software can only alert if the backup job fails or if it generates warnings which is fine.
But what happens when the job doesnt start , the SMVI service doesnt stop and nothing alerts you to the fact that no backups are taken... this was the second time this happened , the first time could be attributed to a random occurrence but the second time it happens ... well then your waiting to get caught with your pants around your ankles...
I made a short script , scheduled to run each evening , which would send a mail if there were no snapshots less than a day old.

#gets todays date , stores it in the format day-month-year in the variable $nowDate
$NowDate =get-date -uformat "%d-%m-%y"

#Loops through c:\snapshotsourcelist.txt assigning one line to the variable $snappath
For ($file = [System.IO.File]::OpenText("c:\snapshotsourcelist.txt");
     !($file.EndOfStream); $snappath = $file.readline()) 
{
#recursively scan through all subdirectories of $snappath
$Snapshotlist = get-childitem $snappath -recurse |
#we are only interested in files and not directories
where-object {$_.mode -notmatch "d"} |
#cant scan this for some reason so excluding it from the search criteria
Where-object {$_.name -notlike "*iegwydc01*"} |
#returns files which have a datestamp of today
where-object {$_.lastwritetime -gt [datetime]::parse($nowDate)}

#if the variable is empty then send a mail
if (!$Snapshotlist) {

write-host 'Snapshot Alert , No Snapshots for' +$snappath 'were taken since 00:00 last night'

        $SmtpClient = new-object system.net.mail.smtpClient
        $SmtpServer = "mailserver.domain.com"
        $SmtpClient.host =
        $SmtpServer

        $From = "ie-dlitalerts@domain.com"
        $To = "Darragh@domain.com"
        $Title = 'Snapshot Alert , No Snapshots for' +$snappath
        $Body = 'Snapshot Alert , No Snapshots for' +$snappath
        $SmtpClient.Send($from,$to,$title,$Body)  }

}

Storage basics , IOPS Penalty and RAID

Post from Yellow bricks showing the write penalty for various raid implementations
http://www.yellow-bricks.com/2009/12/23/iops/

Nice series of blogs to cement the basics ... again http://vmtoday.com/2009/12/storage-basics-part-i-intro/

High %costop values - no CPU contention - Poor performance

High %costop values - no CPU contention - Poor performance

Presented itself as general performance problems in a regional office , specifically one of the application servers was performing very poorly , with frequent application timeouts , exchange was going offline and VM's were becoming orphaned in vSphere , i logged in to a windows server and saw that the CPU was operating at 100% , in performance view in vsphere the server was consuming approximately 300Mhz , this behaviour was repeated on all other servers in the cluster.

In the DRS view i could see that the servers were receiving appx 10% of entitled resources, there were no limits or reservations set on any of the VM's , on examining the CPU counters in ESXTOP i found that all of the servers had extremely high %costop values (~80% - 90%) , this would normally be indicative of over committed CPU resources on SMP VM’s , as ESX throttles individual CPU’s to prevent skew when some CPU’s make progress and others are unable to due to being scheduled on other VM’s. In our case this could not have been the cause as we had more physical CPU’s than vCPU’s.

During the troubleshooting I noticed that we periodically had huge latencies on the storage system , sometimes spiking to 6 seconds , the first strange thing was that the latencies were within acceptable limits until the IOPS rose above 600 , the second strange point was that combined total of CIFS and NFS IOPS were rarely sustained above 400 IOPS.

This had me stumped until i discovered that the storage array had been populated with 7.2K SATA instead of the 15K disks which i expected , immediately i saw why we weren’t seeing at least double the number of IOPS before latencies ramped up , with 8 usable disks we should see 600 IOPS , instead of the 1200 we expected.

The second point was where were the mysterious extra IOPS coming from , after more investigation we found that the Netapp 2020 has hidden aggregate level snapshots which were tipping us well over the 600IOPS threshold , these hidden jobs were set to run every 3 hours , we rescheduled these to run outside of production hours.

The high costop value can be attributed to the fact that the vCPU has to wait for IO completion and as IO completion was taking an extended period of time , ESX was costopping the CPU’s leading to extremely poor performance

PS Script to move users to an OU

AD Functional Account cleanup , i created this script to move a list of users from multiple OUs to an OU
where they will be mass disabled.

import-CSV c:\testdisableduseraccounts.csv -Header @("Name") | foreach-object { Get-ADUser $_.Name | Move-ADObject -Targetpath "ou=temp disabled,ou=disabled user accounts,dc=DC,dc=DC,dc=domain,dc=com" }

unintended repeating keystrokes When accessing VMware console over high latency/low BW connection

power off VM
 Modify VMX file -> keyboard.typematicMinDelay = "2000000" (sets keyboard repeat to 2 seconds)
  (Either through text editor or , edit settings , options , general , configuration parameters , add row)
Power on VM