Edit /etc/nova/nova.conf

The below option specifies whether to start guests that were running before the host rebooted.
Set it to True.

resume_guests_state_on_host_boot = True

You can use guestfish to make changes to your Openstack images such as QCOW2 files.  It's only a yum install away.

In the below example I will change the root password.

Generate an encrypted password:
openssl passwd -1 new_password

Use guestfish to edit /etc/shadow inside the rhel-7-guest image:
guestfish -a rhel-7-server-guest-image.qcow2
> run
> list-filesystems
> mount /dev/sda1 /
> vi /etc/shadow

Relace the root encrypted password in /etc/shadow with the new one you just generated.

Quit guestfish:
> quit

You can now import the new image into Openstack.

I had a very frustrating issue where my router interfaces within Openstack were down.
I did the below to resolve this issue.
vi /etc/neutron/plugins/ml2/ml2_conf.ini

Within the file there is an ml2 section.  I had to modify the type_drivers and mechanism drivers with the below content.
[ml2]
type_drivers=flat,vlan,vxlan
tenant_network_types=vxlan
mechanism_drivers=openvswitch,linuxbridge,l2population
path_mtu=0
extension_drivers=port_security,qos

Then restart the associated services:
systemctl restart neutron-l3-agent.service
systemctl restart neutron-server.service

NOTE! On further inspection the issue may have been that one of the above services was not enabled!

I had another very frustrating issue where I couldn't provision more than one instance with HEAT.
The first instance would provision fine, but the second would fail.
This was not a quota issue.

When creating the stack I would receive:
"No valid host was found. There are not enough hosts available., Code: 500"

Inspection of /var/log/nova/nova-scheduler.log showed:
"Got no allocation candidates from the Placement API. This could be due to insufficient resources or a temporary occurrence as compute nodes start up."

I resolved the problem by changing my HEAT in OS::Nova::Server from "image: Centos 7" to:
block_device_mapping_v2:
       - delete_on_termination: true
         image:       "Centos 7"
         volume_size: 10

The below HEAT files have been used to create this network for Development purposes.

The below heat creates our foundation infrastructure.
A router with two networks.
External allows traffic out to the internet.
Internal is for guest VMs to connect to.

Note, to connect to your guests from Openstack or another PC you will need a persistent static route.

Create /etc/sysconfig/network-scripts/route-br-ex with content like:
192.168.102.0/24 via 192.168.1.31

The first entry is the subnet of your internal network that your guests are on.
The second entry is the IP of your router's interface that is connected to the external network.

heat_template_version: rocky

description: 'linux puppet server and core networking template'

parameters:
  puppetserver:          {type: 'string', default: 'puppetserver.openstacklocal'}
  puppetserver-ip:       {type: 'string', default: '192.168.102.50'}
  linux-admin:           {type: 'string', default: 'admin'}
  linux-admin-password:  {type: 'string', default: 'password'}
  ntp-server:            {type: 'string', default: 'pool.ntp.org'}
  site.pp:
    type: string
    default: |
      node default {
        notify {'Puppet is working with the Puppetserver':}
      }

resources:

  # This network provides access to the Internet
  external_network:
    description: External network   
    type: OS::Neutron::ProviderNet
    properties:
      name: external_network
      network_type: flat
      # extnet was provided in the installer
      physical_network: extnet
      router_external: true

  external_subnet:
    type: OS::Neutron::Subnet
    properties:
      name: external_subnet
      network_id: { get_resource: external_network }
      cidr: 192.168.1.0/24
      allocation_pools: [{"start": "192.168.1.25", "end": "192.168.1.35"}]
      enable_dhcp: false
      gateway_ip: "192.168.1.253"

  edge_router:
    type:                    OS::Neutron::Router
    properties:
      external_gateway_info: { network: external_network }
      name:                  edge_router

  # Our guests will connect to this network
  internal_network0:
    type:              OS::Neutron::Net
    properties:
      admin_state_up: true
      name:            "internal_network0"

  internal_subnet0:
    type:              OS::Neutron::Subnet
    properties:
      name:            "internal_subnet0"
      network:         { get_resource: internal_network0 }
      cidr:            "192.168.102.0/24"
      dns_nameservers: ["8.8.8.8"]
      gateway_ip:      "192.168.102.1"
      ip_version:      4

  router_interface_0:
    type:        OS::Neutron::RouterInterface
    properties:
      router_id: { get_resource: edge_router }
      subnet:    { get_resource: internal_subnet0 }

  puppetserver_sec_group:
    type: OS::Neutron::SecurityGroup
    properties:
      name: puppetserver_sec_group
      rules:
        - protocol: icmp
          direction: ingress
        - protocol: icmp
          direction: egress
        - protocol: udp
          direction: egress
        - protocol: tcp
          direction: ingress
          remote_ip_prefix: { get_attr: [internal_subnet0, cidr] }
          port_range_min: 8140
          port_range_max: 8140
        - protocol: tcp
          direction: egress
          port_range_min: 1
          port_range_max: 65535
        - protocol: tcp
          direction: ingress
          remote_ip_prefix: 0.0.0.0/0
          port_range_min: 22
          port_range_max: 22

  puppetserver_port:
    type:        OS::Neutron::Port
    properties:
      name:      "port1"
      network:   { get_resource: internal_network0 }
      security_groups: [ { get_resource: puppetserver_sec_group } ]
      fixed_ips: [{"subnet": { get_resource: internal_subnet0 }, "ip_address": {get_param: puppetserver-ip}}]

  puppetserver0:
    type:       OS::Nova::Server
    properties:
      name:     {get_param: puppetserver}
      flavor:   m1.medium
      block_device_mapping_v2:
       - delete_on_termination: true
         image:       "centos"
         volume_size: 10
      networks:
        - port: {get_resource: puppetserver_port}
      user_data: 
        str_replace:
          template: |
            #!/bin/bash
            echo -e "minrate=5\ntimeout=500" >> /etc/yum.conf;
            echo -e "$puppetserver-ip $puppetserver puppet" >> /etc/hosts;
            useradd -G wheel -p $(openssl passwd -1 $password) $admin;
            sed -i "s/^PasswordAuthentication no/PasswordAuthentication yes/g" /etc/ssh/sshd_config && systemctl restart sshd;
            timedatectl set-timezone Australia/Sydney;
            yum install -y ntp;
            ntpdate $ntp-server;
            yum install -y https://yum.puppetlabs.com/puppet5/puppet5-release-el-7.noarch.rpm;
            yum install -y puppetserver;
            systemctl enable puppetserver;
            systemctl start puppetserver
            sleep 30;
            /opt/puppetlabs/puppet/bin/puppet config set server $puppetserver;
            /opt/puppetlabs/puppet/bin/puppet config set reports store,log;
            echo -e "*" > /etc/puppetlabs/puppet/autosign.conf;
            echo -e "$site.pp" > /etc/puppetlabs/code/environments/production/manifests/site.pp;
            echo -e "modulepath = modules:site\nconfig_version = '/bin/echo $(date +'%Y-%m-%d %T')'" > /etc/puppetlabs/code/environments/production/environment.conf
            systemctl enable puppet;
            yum update -y
            systemctl start puppet;
          params:
            $puppetserver:    {get_param: puppetserver}
            $puppetserver-ip: {get_param: puppetserver-ip}
            $admin:           {get_param: linux-admin}
            $password:        {get_param: linux-admin-password}
            $ntp-server:      {get_param: ntp-server}
            $site.pp:         {get_param: site.pp}

heat_template_version: rocky

description: 'linux puppet client template'

parameters:
  puppetserver:          {type: 'string', default: 'puppetserver.openstacklocal'}
  puppetserver-ip:       {type: 'string', default: '192.168.102.50'}
  linux-admin:           {type: 'string', default: 'admin'}
  linux-admin-password:  {type: 'string', default: 'password'}
  ntp-server:            {type: 'string', default: 'pool.ntp.org'}
  # Change these values for new clients
  agent:                 {type: 'string', default: 'linuxagent0'}
  agent-port-name:       {type: 'string', default: 'linuxagent0-port'}
  agent-sec-group:       {type: 'string', default: 'linuxagent0-sec-group'}
  agent-ip:              {type: 'string', default: '192.168.102.51'}

resources:

  puppetagent_sec_group:
    type: OS::Neutron::SecurityGroup
    properties:
      name: { get_param: agent-sec-group }
      rules:
        - protocol: icmp
          direction: ingress
        - protocol: icmp
          direction: egress
        - protocol: udp
          direction: egress
        - protocol: tcp
          direction: egress
          port_range_min: 1
          port_range_max: 65535
        - protocol: tcp
          direction: ingress
          port_range_min: 22
          port_range_max: 22

  agent_port:
    type:        OS::Neutron::Port
    properties:
      name:      { get_param: agent-port-name }
      security_groups: [ { get_resource: puppetagent_sec_group } ]
      network: internal_network0
      fixed_ips: [{"subnet": "internal_subnet0", "ip_address": {get_param: agent-ip}}]

  linuxagent0:
    type:           OS::Nova::Server
    depends_on:     agent_port
    properties:
      name:         { get_param: agent }
      flavor:       "m1.small"
      block_device_mapping_v2:
       - delete_on_termination: true
         image:       "centos"
         volume_size: 10
      networks:
        - port:     { get_resource: agent_port }
      user_data:
        str_replace:
          template: |
            #!/bin/bash
            echo -e "minrate=5\ntimeout=500" >> /etc/yum.conf;
            echo -e "$puppetserver-ip $puppetserver puppet" >> /etc/hosts;
            useradd -G wheel -p $(openssl passwd -1 $password) $admin;
            sed -i "s/^PasswordAuthentication no/PasswordAuthentication yes/g" /etc/ssh/sshd_config && systemctl restart sshd;
            timedatectl set-timezone Australia/Sydney;
            yum install -y ntp;
            ntpdate $ntp-server;
            yum install -y https://yum.puppetlabs.com/puppet5/puppet5-release-el-7.noarch.rpm;
            yum install -y puppet-agent
            systemctl enable puppet;
            sleep 60;
            /opt/puppetlabs/puppet/bin/puppet config set server $puppetserver;
            yum update -y
            systemctl start puppet;
          params:
            $puppetserver:    {get_param: puppetserver}
            $puppetserver-ip: {get_param: puppetserver-ip}
            $ntp-server:      {get_param: ntp-server}
            $admin:           {get_param: linux-admin}
            $password:        {get_param: linux-admin-password}

heat_template_version: rocky

description: 'windows puppet client template'

parameters:
  puppetserver:          {type: 'string', default: 'puppet'}
  puppetserver-ip:       {type: 'string', default: '192.168.102.50'}
  win-user:              {type: 'string', default: 'user'}
  win-user-password:     {type: 'string', default: 'password'}
  win-admin-password:    {type: 'string', default: 'password'}
  ntp-server:            {type: 'string', default: 'pool.ntp.org'}
  # Change these values for new clients
  agent:                 {type: 'string', default: 'winagent0'}
  agent-port-name:       {type: 'string', default: 'winagent0-port'}
  agent-sec-group:       {type: 'string', default: 'winagent0-sec-group'}
  agent-ip:              {type: 'string', default: '192.168.102.52'}

resources:

  puppetagent_win_sec_group:
    type: OS::Neutron::SecurityGroup
    properties:
      name: puppetagent_win_sec_group
      rules:
        - protocol:  icmp
          direction: ingress
        - protocol:  icmp
          direction: egress
        - protocol:  udp
          direction: egress
        - protocol:  tcp
          direction: egress
          port_range_min: 1
          port_range_max: 65535
        - protocol:  tcp
          direction: ingress
          port_range_min: 3389
          port_range_max: 3389

  win_agent_port:
    type:        OS::Neutron::Port
    properties:
      name:            { get_param: agent-port-name }
      security_groups: [ { get_resource: puppetagent_win_sec_group } ]
      network:         internal_network0
      fixed_ips:       [{"subnet": "internal_subnet0", "ip_address": {get_param: agent-ip}}]

  win-agent:
    type:           OS::Nova::Server
    depends_on:     win_agent_port
    properties:
      name:         { get_param: agent }
      flavor:       m1.medium
      block_device_mapping_v2:
       - delete_on_termination: true
         image:                 "windows"
         volume_size:           40
      networks:
        - port:     {get_resource: win_agent_port}
      metadata:
        admin_pass: {get_param: win-admin-password}
      user_data: 
        str_replace:
          template: |
            #ps1_sysnative
            Rename-Computer -newname $agent
            w32tm /config /manualpeerlist:$ntp-server /syncfromflags:manual /reliable:yes /update
            net stop w32time
            net start w32time
            w32tm /resync; w32tm /resync; w32tm /resync; w32tm /resync; w32tm /resync
            Add-Content -Path C:\Windows\System32\drivers\etc\hosts -Value "$puppetserver-ip`t$puppetserver" -Force
            cmd /c puppet config set server $puppetserver
            (Get-WmiObject Win32_TerminalServiceSetting -Namespace root\cimv2\TerminalServices).SetAllowTsConnections(1,1) | Out-Null
            (Get-WmiObject -Class "Win32_TSGeneralSetting" -Namespace root\cimv2\TerminalServices -Filter "TerminalName='RDP-tcp'").SetUserAuthenticationRequired(0) | Out-Null
            Get-NetFirewallRule -DisplayName "Remote Desktop*" | Set-NetFirewallRule -enabled true
            $password=ConvertTo-SecureString -AsPlainText -Force $win-user-password
            New-LocalUser -Name $win-user -Password $password; Add-LocalGroupMember -Group "Administrators" -Member $win-user
            Disable-LocalUser -Name "Administrator"
            Restart-Computer -force
          params:
            $win-user:          {get_param: win-user}
            $agent:             {get_param: agent}
            $win-user-password: {get_param: win-user-password}
            $puppetserver:      {get_param: puppetserver}
            $puppetserver-ip:   {get_param: puppetserver-ip}
            $ntp-server:        {get_param: ntp-server}